Popular
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139// Usage: deno run --allow-net --allow-write update.ts 14import { Emoji } from "./types.ts";
const Qualification = { FULLY_QUALIFIED: "fully-qualified", MINIMALLY_QUALIFIED: "minimally-qualified", UNQUALIFIED: "unqualified", COMPONENT: "component",} as const;
type Qualification = typeof Qualification[keyof typeof Qualification];
interface FullEmoji extends Emoji { codepoints: string[]; ln: string; qualification: Qualification;}
const version = Number(Deno.args[0]);const isPost13: boolean = version >= 13;
const normalizeName = (desc: string): string => desc.replace(/[^a-zA-Z0-9_*#]+/g, "_").toLowerCase();
// Prior to version 14, emojiText lines do not include version infoconst parseLineRegex = isPost13 ? /^(.*?);(.*?)#(.*?)E(\S*?)\s(.*)/i : /^(.*?);(.*?)#\s(.*?)\s(.*)/;
// Based on: https://unicode.org/reports/tr51/proposed.html#Versioning// When emoji version > 5, it begins to match with unicode version.const emojiUnicodeMap: { [emojiVersion: number]: number } = { [0]: 0, [0.6]: 6, [0.7]: 7, [1.0]: 8, [2.0]: 8, [3.0]: 9, [4.0]: 9, [5.0]: 10,};
// Since we are only using CLDR Annotations to decorate, fetch latest.// Use full list; derived list is skin-tone onlyconst annotationsURL = "https://raw.githubusercontent.com/unicode-org/cldr-json/main/cldr-json/cldr-annotations-full/annotations/en/annotations.json";const annotationsResp = await fetch(annotationsURL);const annotations: { [emoji: string]: { default: string[]; tts: string[] };} = (await annotationsResp.json()).annotations.annotations;
const displayVersion: string = version.toFixed(1);const emojiTestURL = `https://unicode.org/Public/emoji/${displayVersion}/emoji-test.txt`;const emojiTestResp = await fetch(emojiTestURL);const emojiTestText = await emojiTestResp.text();
// Store all tags, so we can make sure alias are not dupedconst allTags = new Set();const dupeTags = new Set();
const allEmojis: Emoji[] = emojiTestText.split("# group: ").slice(1) .flatMap(function parseGroups(groupText: string) { const [groupNameText, ...subgroups] = groupText.split("# subgroup: "); const groupName = groupNameText.replace("\n\n", ""); return subgroups.flatMap(function parseSubgroups(subgroupText: string) { const [subgroupName, ...emojis] = subgroupText.split("\n"); return emojis.flatMap( function parseEmojiFromLine(ln: string): FullEmoji { const lineData = ln.match(parseLineRegex) || []; const codepoints = (lineData[1] || "").trim().split(/\s+/); const qualification = (lineData[2] || "").trim() as Qualification; const emojiChar = (lineData[3] || "").trim(); const description = isPost13 ? lineData[5] : lineData[4]; const emojiVersion = isPost13 ? Number(lineData[4]) : 0; const unicodeVersion = emojiUnicodeMap[emojiVersion] != null ? emojiUnicodeMap[emojiVersion] : emojiVersion;
const tags = (annotations[emojiChar]?.default || []); if (description) tags.push(description);
tags.forEach((tag) => { const name = normalizeName(tag); allTags.has(name) ? dupeTags.add(name) : allTags.add(name); });
return { ln, codepoints, qualification, group: groupName, subgroup: subgroupName, emoji: emojiChar, emojiVersion, unicodeVersion, description, tags: Array.from(new Set(tags)), // dedupe if desc is a tag aliases: [], }; }, ); }); }) // Effects that rely on data from other emojis .filter((emoji: FullEmoji, index: number, emojis: FullEmoji[]) => { const { aliases, codepoints, description, ln, qualification, tags } = emoji;
if (ln.trim() === "" || ln[0] === "#") return false; if (!codepoints.length || !emoji.emoji || !description) return false; // Remove unqualified, partically-qualified, component if (qualification !== Qualification.FULLY_QUALIFIED) return false;
tags.forEach((tag) => { const name = normalizeName(tag); if (!dupeTags.has(name)) aliases.push(name); });
// Handle aliases after populating tag sets for if (!aliases.length) aliases.push(normalizeName(description));
// Remove emojis with skin tones // Skin tones always follow base emoji, so if the prev matches, // Consider it to have a skin tone. if (/skin tone/.test(description)) { const prev = emojis[index - 1]; if (codepoints.indexOf(prev?.codepoints?.[0]) !== -1) { prev.skinTones = true; } return false; }
return true; }) // deno-lint-ignore no-unused-vars .map(({ ln, codepoints, qualification, ...e }: FullEmoji): Emoji => e);
Deno.writeTextFileSync(`./all.json`, JSON.stringify(allEmojis, null, 2));