From b09af7fc07164c3f0a8f8721053e48d6f5ad0465 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 25 Apr 2022 17:09:32 -0600 Subject: [PATCH 1/3] noto-tools: adjust dependencies to allow running notodiff --- pkgs/data/fonts/noto-fonts/tools.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/data/fonts/noto-fonts/tools.nix b/pkgs/data/fonts/noto-fonts/tools.nix index 7d850ef716674..d43432c8ed579 100644 --- a/pkgs/data/fonts/noto-fonts/tools.nix +++ b/pkgs/data/fonts/noto-fonts/tools.nix @@ -3,7 +3,7 @@ , defcon, fontmath, fontparts, fontpens, fonttools, lxml , mutatormath, pathspec, psautohint, pyclipper, pytz, regex, scour , toml, typed-ast, ufonormalizer, ufoprocessor, unicodedata2, zopfli -, pillow, six, bash, setuptools-scm }: +, pillow, six, bash, setuptools-scm, setuptools }: buildPythonPackage rec { pname = "nototools"; @@ -40,11 +40,13 @@ buildPythonPackage rec { lxml mutatormath pathspec + pillow psautohint pyclipper pytz regex scour + setuptools toml typed-ast ufonormalizer @@ -54,7 +56,6 @@ buildPythonPackage rec { ]; checkInputs = [ - pillow six bash ]; From f4cad5a54855ad872a5e7ca38df0b5d1de8c8f53 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 25 Apr 2022 17:10:29 -0600 Subject: [PATCH 2/3] noto-fonts: 2020-01-23 -> unstable-2022-04-25 and switch to variable fonts --- nixos/tests/noto-fonts.nix | 2 + pkgs/data/fonts/noto-fonts/analyze_scan.py | 156 +++++++++++++++++++++ pkgs/data/fonts/noto-fonts/default.nix | 61 ++++---- 3 files changed, 195 insertions(+), 24 deletions(-) create mode 100644 pkgs/data/fonts/noto-fonts/analyze_scan.py diff --git a/nixos/tests/noto-fonts.nix b/nixos/tests/noto-fonts.nix index e4c33fe26a9e6..7302082edd2ae 100644 --- a/nixos/tests/noto-fonts.nix +++ b/nixos/tests/noto-fonts.nix @@ -26,6 +26,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { testScript = # extracted from http://www.clagnut.com/blog/2380/ + # emoji are from Unicode 14 (latest standard) let testText = builtins.toFile "test.txt" '' the quick brown fox jumps over the lazy dog 視野無限廣，窗外有藍天 @@ -33,6 +34,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { いろはにほへと ちりぬるを わかよたれそ つねならむ うゐのおくやま けふこえて あさきゆめみし ゑひもせす 다람쥐 헌 쳇바퀴에 타고파 中国智造，慧及全球 + 🫠🫣🫱🏻‍🫲🏿 ''; in '' machine.wait_for_x() diff --git a/pkgs/data/fonts/noto-fonts/analyze_scan.py b/pkgs/data/fonts/noto-fonts/analyze_scan.py new file mode 100644 index 0000000000000..ffd31f6148857 --- /dev/null +++ b/pkgs/data/fonts/noto-fonts/analyze_scan.py @@ -0,0 +1,156 @@ +import csv +import re +from collections import defaultdict + +# fc-scan ./hinted ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv + +blacklist = [ + # provides "Noto S Display, M" while main font provides "Noto S, Display M". + # package both by blacklisting and force-adding Display fonts later + # https://github.com/googlefonts/noto-fonts/issues/2315 + r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-Italic-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSerifDisplay-VF\.ttf", + # provides "Noto Sans Display X, Regular" instead of "Noto Sans Display, X" + # https://github.com/googlefonts/noto-fonts/issues/2315 + r"\./unhinted/otf/NotoSansDisplay/NotoSansDisplay-[A-Za-z]+\.otf", + # provides "Noto Sans X" instead of "Noto Sans X UI" + # https://github.com/googlefonts/noto-fonts/issues/2316 + r"\./unhinted/(slim-)?variable-ttf/NotoSansKannadaUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansMalayalamUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansSinhalaUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTeluguUI-VF\.ttf", + # extraneous files + # comment in https://github.com/googlefonts/noto-fonts/issues/2316 + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamil-android-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-android-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF-torecover\.ttf", + # provides "FontName" instead of "Font Name" + # https://github.com/googlefonts/noto-fonts/issues/2317 + r"\./unhinted/(slim-)?variable-ttf/NotoSansMeeteiMayek-VF\.ttf", + r"\./unhinted/(slim-)?variable-ttf/NotoSerifTamilSlanted-VF\.ttf", + # font names are missing Bold, Italic, etc. + # https://github.com/googlefonts/noto-fonts/issues/2317 + r"\./unhinted/(slim-)?variable-ttf/NotoSansHanifiRohingya-VF\.ttf", + # duplicate fonts in NotoSansTifinagh + # https://github.com/googlefonts/noto-fonts/issues/2326 + r"\./unhinted/otf/NotoSansTifinagh/NotoSansTifinagh[A-Z].*\.otf" +] + +blacklist = list(map(re.compile, blacklist)) + +class Font: + def __init__(self): + self.hinted_ttf = None + self.unhinted_ttf = None + self.otf = None + self.vf = None + self.slimvf = None + self.extra = None + +fonts = defaultdict(Font) + +# KDE uses Light/LightItalic, so keep those in the main package, see +# https://mail.kde.org/pipermail/distributions/2017-November/000706.html +# https://mail.kde.org/pipermail/distributions/2017-November/000709.html +extrafontpattern = re.compile(r".*(Black|Condensed|Extra|Medium|Semi|Thin).*") + +with open('scan.csv', newline='') as csvfile: + scanreader = csv.reader(csvfile, delimiter=',', quotechar='"') + for row in scanreader: + [font, fontfile] = row + if any(rgx.match(fontfile) for rgx in blacklist): + continue + if fontfile.startswith("./hinted/ttf"): + if fonts[font].hinted_ttf and fonts[font].hinted_ttf != fontfile: + print("duplicate httf", font, ":", fonts[font].hinted_ttf, "and", fontfile) + fonts[font].hinted_ttf = fontfile + elif fontfile.startswith("./unhinted/ttf"): + if fonts[font].unhinted_ttf and fonts[font].unhinted_ttf != fontfile: + print("duplicate uttf", font, ":", fonts[font].unhinted_ttf, "and", fontfile) + fonts[font].unhinted_ttf = fontfile + elif fontfile.startswith("./unhinted/otf"): + if fonts[font].otf and fonts[font].otf != fontfile: + print("duplicate otf", font, ":", fonts[font].otf, "and", fontfile) + fonts[font].otf = fontfile + elif fontfile.startswith("./unhinted/variable-ttf"): + if fonts[font].vf and fonts[font].vf != fontfile and font != "": + print("duplicate vf", font, ":", fonts[font].vf, "and", fontfile) + fonts[font].vf = fontfile + elif fontfile.startswith("./unhinted/slim-variable-ttf"): + if fonts[font].slimvf and fonts[font].slimvf != fontfile and font != "": + print("duplicate slim", font, ":", fonts[font].slimvf, "and", fontfile) + fonts[font].slimvf = fontfile + else: + print("Unknown font directory: ", ', '.join(row)) + raise NotImplementedError + + fonts[font].extra = extrafontpattern.match(font) != None or font == "" + +# inconsistent names, see https://github.com/googlefonts/noto-fonts/issues/2317 +# for these names the VF names seem acceptable +lightlightpattern = re.compile(r"Noto Sans Hebrew (New )?Light Light") +hmongpattern = re.compile(r"Noto Serif Nyiakeng Puachue Hmong .+") + +# compute noto-font files + +mode = "vf" # "otf" "slimvf" "vf" +modeextra = "vf" # "otf" "vf" + +# force-add Display VF's to file list, see above +# https://github.com/googlefonts/noto-fonts/issues/2315 +filelist = set( + ["./unhinted/variable-ttf/NotoSansDisplay-Italic-VF.ttf", + "./unhinted/variable-ttf/NotoSansDisplay-VF.ttf", + "./unhinted/variable-ttf/NotoSerifDisplay-VF.ttf"]) + +for name, font in fonts.items(): + if font.extra: + continue + + if font.vf and mode == "vf": + if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name): + print("missing otf:", name, ", ", font.vf) + continue + filelist.add(font.vf) + elif font.slimvf and mode == "slimvf": + if not font.otf and font.slimvf != "": + print("missing otf:", name, ", ", font.slimvf) + continue + filelist.add(font.slimvf) + elif font.otf: + filelist.add(font.otf) + elif font.vf: + continue + else: + print("weird font: ", name, ", ", font) + raise NotImplementedError + +with open('noto_fonts_list.txt', 'w') as f: + for item in filelist: + f.write("%s\n" % item) + +extra_list = set() + +for name, font in fonts.items(): + if font.vf in filelist or font.slimvf in filelist or font.otf in filelist: + continue + + if font.vf and modeextra != "otf": + if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name): + print("missing otf:", name, ", ", font.vf) + continue + extra_list.add(font.vf) + elif font.otf: + extra_list.add(font.otf) + elif font.vf: + continue + else: + print("weird font: ", name, ", ", font) + raise NotImplementedError + +with open('noto_extra_list.txt', 'w') as f: + for item in extra_list: + f.write("%s\n" % item) + diff --git a/pkgs/data/fonts/noto-fonts/default.nix b/pkgs/data/fonts/noto-fonts/default.nix index 4d91bf8905d55..3f4f8c0cfae5f 100644 --- a/pkgs/data/fonts/noto-fonts/default.nix +++ b/pkgs/data/fonts/noto-fonts/default.nix @@ -11,32 +11,44 @@ , imagemagick , zopfli , buildPackages +, fontconfig +, python3 }: let - mkNoto = { pname, weights }: + noto-pkg = stdenvNoCC.mkDerivation { - inherit pname; - version = "2020-01-23"; + pname = "noto-fonts"; + version = "unstable-2022-04-25"; src = fetchFromGitHub { owner = "googlefonts"; repo = "noto-fonts"; - rev = "f4726a2ec36169abd02a6d8abe67c8ff0236f6d8"; - sha256 = "0zc1r7zph62qmvzxqfflsprazjf6x1qnwc2ma27kyzh6v36gaykw"; + rev = "a19de47f845dbd4c61b884c7ff90ce993555d05d"; + sparseCheckout = '' + unhinted/otf + unhinted/variable-ttf + ''; + hash = "sha256-7Jzo7402CiOcn2q++zbAzn2tP/y2x7QJuwjcP1NoyLQ="; }; + analyzeScan = ./analyze_scan.py; + + nativeBuildInputs = [ buildPackages.fontconfig.bin buildPackages.python3 ]; + + outputs = [ "out" "extra" ]; + installPhase = '' - # We copy in reverse preference order -- unhinted first, then - # hinted -- to get the "best" version of each font while - # maintaining maximum coverage. - # - # TODO: install OpenType, variable versions? - local out_ttf=$out/share/fonts/truetype/noto - install -m444 -Dt $out_ttf phaseIII_only/unhinted/ttf/*/*-${weights}.ttf - install -m444 -Dt $out_ttf phaseIII_only/hinted/ttf/*/*-${weights}.ttf - install -m444 -Dt $out_ttf unhinted/*/*-${weights}.ttf - install -m444 -Dt $out_ttf hinted/*/*-${weights}.ttf + fc-scan ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv + python $analyzeScan + local out_ttf=$out/share/fonts/google-noto + while read p; do + install -m444 -Dt $out_ttf "$p" + done Date: Mon, 25 Apr 2022 19:18:03 -0600 Subject: [PATCH 3/3] noto-fonts: split out croscore --- pkgs/data/fonts/noto-fonts/analyze_scan.py | 6 +++++- pkgs/data/fonts/noto-fonts/default.nix | 11 +++++++++-- pkgs/top-level/all-packages.nix | 5 +++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pkgs/data/fonts/noto-fonts/analyze_scan.py b/pkgs/data/fonts/noto-fonts/analyze_scan.py index ffd31f6148857..53007aedb9746 100644 --- a/pkgs/data/fonts/noto-fonts/analyze_scan.py +++ b/pkgs/data/fonts/noto-fonts/analyze_scan.py @@ -35,7 +35,11 @@ r"\./unhinted/(slim-)?variable-ttf/NotoSansHanifiRohingya-VF\.ttf", # duplicate fonts in NotoSansTifinagh # https://github.com/googlefonts/noto-fonts/issues/2326 - r"\./unhinted/otf/NotoSansTifinagh/NotoSansTifinagh[A-Z].*\.otf" + r"\./unhinted/otf/NotoSansTifinagh/NotoSansTifinagh[A-Z].*\.otf", + # croscore fonts, packaged separately + r"\./unhinted/otf/Arimo/.*", + r"\./unhinted/otf/Cousine/.*", + r"\./unhinted/otf/Tinos/.*" ] blacklist = list(map(re.compile, blacklist)) diff --git a/pkgs/data/fonts/noto-fonts/default.nix b/pkgs/data/fonts/noto-fonts/default.nix index 3f4f8c0cfae5f..24cc9189153fe 100644 --- a/pkgs/data/fonts/noto-fonts/default.nix +++ b/pkgs/data/fonts/noto-fonts/default.nix @@ -36,7 +36,7 @@ let nativeBuildInputs = [ buildPackages.fontconfig.bin buildPackages.python3 ]; - outputs = [ "out" "extra" ]; + outputs = [ "out" "extra" "croscore" ]; installPhase = '' fc-scan ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv @@ -49,6 +49,10 @@ let while read p; do install -m444 -Dt $extra_ttf "$p" done