diff --git a/nixos/tests/noto-fonts.nix b/nixos/tests/noto-fonts.nix index e4c33fe26a9e6..7302082edd2ae 100644 --- a/nixos/tests/noto-fonts.nix +++ b/nixos/tests/noto-fonts.nix @@ -26,6 +26,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { testScript = # extracted from http://www.clagnut.com/blog/2380/ + # emoji are from Unicode 14 (latest standard) let testText = builtins.toFile "test.txt" '' the quick brown fox jumps over the lazy dog 視野無限廣，窗外有藍天 @@ -33,6 +34,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { いろはにほへと ちりぬるを わかよたれそ つねならむ うゐのおくやま けふこえて あさきゆめみし ゑひもせす 다람쥐 헌 쳇바퀴에 타고파 中国智造，慧及全球 + 🫠🫣🫱🏻‍🫲🏿 ''; in '' machine.wait_for_x() diff --git a/pkgs/data/fonts/noto-fonts/analyze_scan.py b/pkgs/data/fonts/noto-fonts/analyze_scan.py new file mode 100644 index 0000000000000..53007aedb9746 --- /dev/null +++ b/pkgs/data/fonts/noto-fonts/analyze_scan.py @@ -0,0 +1,160 @@ +import csv +import re +from collections import defaultdict + +# fc-scan ./hinted ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv + +blacklist = [ + # provides "Noto S Display, M" while main font provides "Noto S, Display M". + # package both by blacklisting and force-adding Display fonts later + # https://github.com/googlefonts/noto-fonts/issues/2315 + r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-Italic-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSerifDisplay-VF\.ttf", + # provides "Noto Sans Display X, Regular" instead of "Noto Sans Display, X" + # https://github.com/googlefonts/noto-fonts/issues/2315 + r"\./unhinted/otf/NotoSansDisplay/NotoSansDisplay-[A-Za-z]+\.otf", + # provides "Noto Sans X" instead of "Noto Sans X UI" + # https://github.com/googlefonts/noto-fonts/issues/2316 + r"\./unhinted/(slim-)?variable-ttf/NotoSansKannadaUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansMalayalamUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansSinhalaUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTeluguUI-VF\.ttf", + # extraneous files + # comment in https://github.com/googlefonts/noto-fonts/issues/2316 + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamil-android-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-android-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF-torecover\.ttf", + # provides "FontName" instead of "Font Name" + # https://github.com/googlefonts/noto-fonts/issues/2317 + r"\./unhinted/(slim-)?variable-ttf/NotoSansMeeteiMayek-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSerifTamilSlanted-VF\.ttf", + # font names are missing Bold, Italic, etc. + # https://github.com/googlefonts/noto-fonts/issues/2317 + r"\./unhinted/(slim-)?variable-ttf/NotoSansHanifiRohingya-VF\.ttf", + # duplicate fonts in NotoSansTifinagh + # https://github.com/googlefonts/noto-fonts/issues/2326 + r"\./unhinted/otf/NotoSansTifinagh/NotoSansTifinagh[A-Z].*\.otf", + # croscore fonts, packaged separately + r"\./unhinted/otf/Arimo/.*", + r"\./unhinted/otf/Cousine/.*", + r"\./unhinted/otf/Tinos/.*" +] + +blacklist = list(map(re.compile, blacklist)) + +class Font: + def __init__(self): + self.hinted_ttf = None + self.unhinted_ttf = None + self.otf = None + self.vf = None + self.slimvf = None + self.extra = None + +fonts = defaultdict(Font) + +# KDE uses Light/LightItalic, so keep those in the main package, see +# https://mail.kde.org/pipermail/distributions/2017-November/000706.html +# https://mail.kde.org/pipermail/distributions/2017-November/000709.html +extrafontpattern = re.compile(r".*(Black|Condensed|Extra|Medium|Semi|Thin).*") + +with open('scan.csv', newline='') as csvfile: + scanreader = csv.reader(csvfile, delimiter=',', quotechar='"') + for row in scanreader: + [font, fontfile] = row + if any(rgx.match(fontfile) for rgx in blacklist): + continue + if fontfile.startswith("./hinted/ttf"): + if fonts[font].hinted_ttf and fonts[font].hinted_ttf != fontfile: + print("duplicate httf", font, ":", fonts[font].hinted_ttf, "and", fontfile) + fonts[font].hinted_ttf = fontfile + elif fontfile.startswith("./unhinted/ttf"): + if fonts[font].unhinted_ttf and fonts[font].unhinted_ttf != fontfile: + print("duplicate uttf", font, ":", fonts[font].unhinted_ttf, "and", fontfile) + fonts[font].unhinted_ttf = fontfile + elif fontfile.startswith("./unhinted/otf"): + if fonts[font].otf and fonts[font].otf != fontfile: + print("duplicate otf", font, ":", fonts[font].otf, "and", fontfile) + fonts[font].otf = fontfile + elif fontfile.startswith("./unhinted/variable-ttf"): + if fonts[font].vf and fonts[font].vf != fontfile and font != "": + print("duplicate vf", font, ":", fonts[font].vf, "and", fontfile) + fonts[font].vf = fontfile + elif fontfile.startswith("./unhinted/slim-variable-ttf"): + if fonts[font].slimvf and fonts[font].slimvf != fontfile and font != "": + print("duplicate slim", font, ":", fonts[font].slimvf, "and", fontfile) + fonts[font].slimvf = fontfile + else: + print("Unknown font directory: ", ', '.join(row)) + raise NotImplementedError + + fonts[font].extra = extrafontpattern.match(font) != None or font == "" + +# inconsistent names, see https://github.com/googlefonts/noto-fonts/issues/2317 +# for these names the VF names seem acceptable +lightlightpattern = re.compile(r"Noto Sans Hebrew (New )?Light Light") +hmongpattern = re.compile(r"Noto Serif Nyiakeng Puachue Hmong .+") + +# compute noto-font files + +mode = "vf" # "otf" "slimvf" "vf" +modeextra = "vf" # "otf" "vf" + +# force-add Display VF's to file list, see above +# https://github.com/googlefonts/noto-fonts/issues/2315 +filelist = set( + ["./unhinted/variable-ttf/NotoSansDisplay-Italic-VF.ttf", + "./unhinted/variable-ttf/NotoSansDisplay-VF.ttf", + "./unhinted/variable-ttf/NotoSerifDisplay-VF.ttf"]) + +for name, font in fonts.items(): + if font.extra: + continue + + if font.vf and mode == "vf": + if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name): + print("missing otf:", name, ", ", font.vf) + continue + filelist.add(font.vf) + elif font.slimvf and mode == "slimvf": + if not font.otf and font.slimvf != "": + print("missing otf:", name, ", ", font.slimvf) + continue + filelist.add(font.slimvf) + elif font.otf: + filelist.add(font.otf) + elif font.vf: + continue + else: + print("weird font: ", name, ", ", font) + raise NotImplementedError + +with open('noto_fonts_list.txt', 'w') as f: + for item in filelist: + f.write("%s\n" % item) + +extra_list = set() + +for name, font in fonts.items(): + if font.vf in filelist or font.slimvf in filelist or font.otf in filelist: + continue + + if font.vf and modeextra != "otf": + if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name): + print("missing otf:", name, ", ", font.vf) + continue + extra_list.add(font.vf) + elif font.otf: + extra_list.add(font.otf) + elif font.vf: + continue + else: + print("weird font: ", name, ", ", font) + raise NotImplementedError + +with open('noto_extra_list.txt', 'w') as f: + for item in extra_list: + f.write("%s\n" % item) + diff --git a/pkgs/data/fonts/noto-fonts/default.nix b/pkgs/data/fonts/noto-fonts/default.nix index 4d91bf8905d55..24cc9189153fe 100644 --- a/pkgs/data/fonts/noto-fonts/default.nix +++ b/pkgs/data/fonts/noto-fonts/default.nix @@ -11,32 +11,48 @@ , imagemagick , zopfli , buildPackages +, fontconfig +, python3 }: let - mkNoto = { pname, weights }: + noto-pkg = stdenvNoCC.mkDerivation { - inherit pname; - version = "2020-01-23"; + pname = "noto-fonts"; + version = "unstable-2022-04-25"; src = fetchFromGitHub { owner = "googlefonts"; repo = "noto-fonts"; - rev = "f4726a2ec36169abd02a6d8abe67c8ff0236f6d8"; - sha256 = "0zc1r7zph62qmvzxqfflsprazjf6x1qnwc2ma27kyzh6v36gaykw"; + rev = "a19de47f845dbd4c61b884c7ff90ce993555d05d"; + sparseCheckout = '' + unhinted/otf + unhinted/variable-ttf + ''; + hash = "sha256-7Jzo7402CiOcn2q++zbAzn2tP/y2x7QJuwjcP1NoyLQ="; }; + analyzeScan = ./analyze_scan.py; + + nativeBuildInputs = [ buildPackages.fontconfig.bin buildPackages.python3 ]; + + outputs = [ "out" "extra" "croscore" ]; + installPhase = '' - # We copy in reverse preference order -- unhinted first, then - # hinted -- to get the "best" version of each font while - # maintaining maximum coverage. - # - # TODO: install OpenType, variable versions? - local out_ttf=$out/share/fonts/truetype/noto - install -m444 -Dt $out_ttf phaseIII_only/unhinted/ttf/*/*-${weights}.ttf - install -m444 -Dt $out_ttf phaseIII_only/hinted/ttf/*/*-${weights}.ttf - install -m444 -Dt $out_ttf unhinted/*/*-${weights}.ttf - install -m444 -Dt $out_ttf hinted/*/*-${weights}.ttf + fc-scan ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv + python $analyzeScan + local out_ttf=$out/share/fonts/google-noto + while read p; do + install -m444 -Dt $out_ttf "$p" + done