noto-fonts: 2020-01-23 -> unstable-2022-04-25 and other changes

nixos/tests/noto-fonts.nix

@@ -26,13 +26,15 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
 
   testScript =
     # extracted from http://www.clagnut.com/blog/2380/
+    # emoji are from Unicode 14 (latest standard)
     let testText = builtins.toFile "test.txt" ''
       the quick brown fox jumps over the lazy dog
       視野無限廣，窗外有藍天
       Eĥoŝanĝo ĉiuĵaŭde.
       いろはにほへと ちりぬるを わかよたれそ つねならむ うゐのおくやま けふこえて あさきゆめみし ゑひもせす
       다람쥐 헌 쳇바퀴에 타고파
       中国智造，慧及全球
+      🫠🫣🫱🏻‍🫲🏿
     ''; in
     ''
       machine.wait_for_x()

pkgs/data/fonts/noto-fonts/analyze_scan.py

@@ -0,0 +1,156 @@
+import csv
+import re
+from collections import defaultdict
+
+# fc-scan ./hinted ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv
+
+blacklist = [
+  # provides "Noto S Display, M" while main font provides "Noto S, Display M".
+  # package both by blacklisting and force-adding Display fonts later
+  # https://github.com/googlefonts/noto-fonts/issues/2315
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-Italic-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansDisplay-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSerifDisplay-VF\.ttf",
+  # provides "Noto Sans Display X, Regular" instead of "Noto Sans Display, X"
+  # https://github.com/googlefonts/noto-fonts/issues/2315
+  r"\./unhinted/otf/NotoSansDisplay/NotoSansDisplay-[A-Za-z]+\.otf",
+  # provides "Noto Sans X" instead of "Noto Sans X UI"
+  # https://github.com/googlefonts/noto-fonts/issues/2316
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansKannadaUI-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansMalayalamUI-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansSinhalaUI-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansTeluguUI-VF\.ttf",
+  # extraneous files
+  # comment in https://github.com/googlefonts/noto-fonts/issues/2316
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansTamil-android-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-android-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansTamilUI-VF-torecover\.ttf",
+  # provides "FontName" instead of "Font Name"
+  # https://github.com/googlefonts/noto-fonts/issues/2317
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansMeeteiMayek-VF\.ttf",
+  r"\./unhinted/(slim-)?variable-ttf/NotoSerifTamilSlanted-VF\.ttf",
+  # font names are missing Bold, Italic, etc.
+  # https://github.com/googlefonts/noto-fonts/issues/2317
+  r"\./unhinted/(slim-)?variable-ttf/NotoSansHanifiRohingya-VF\.ttf",
+  # duplicate fonts in NotoSansTifinagh
+  # https://github.com/googlefonts/noto-fonts/issues/2326
+  r"\./unhinted/otf/NotoSansTifinagh/NotoSansTifinagh[A-Z].*\.otf"
+]
+
+blacklist = list(map(re.compile, blacklist))
+
+class Font:
+  def __init__(self):
+    self.hinted_ttf = None
+    self.unhinted_ttf = None
+    self.otf = None
+    self.vf = None
+    self.slimvf = None
+    self.extra = None
+
+fonts = defaultdict(Font)
+
+# KDE uses Light/LightItalic, so keep those in the main package, see
+# https://mail.kde.org/pipermail/distributions/2017-November/000706.html
+# https://mail.kde.org/pipermail/distributions/2017-November/000709.html
+extrafontpattern = re.compile(r".*(Black|Condensed|Extra|Medium|Semi|Thin).*")
+
+with open('scan.csv', newline='') as csvfile:
+  scanreader = csv.reader(csvfile, delimiter=',', quotechar='"')
+  for row in scanreader:
+    [font, fontfile] = row
+    if any(rgx.match(fontfile) for rgx in blacklist):
+      continue
+    if fontfile.startswith("./hinted/ttf"):
+      if fonts[font].hinted_ttf and fonts[font].hinted_ttf != fontfile:
+        print("duplicate httf", font, ":", fonts[font].hinted_ttf, "and", fontfile)
+      fonts[font].hinted_ttf = fontfile
+    elif fontfile.startswith("./unhinted/ttf"):
+      if fonts[font].unhinted_ttf and fonts[font].unhinted_ttf != fontfile:
+        print("duplicate uttf", font, ":", fonts[font].unhinted_ttf, "and", fontfile)
+      fonts[font].unhinted_ttf = fontfile
+    elif fontfile.startswith("./unhinted/otf"):
+      if fonts[font].otf and fonts[font].otf != fontfile:
+        print("duplicate otf", font, ":", fonts[font].otf, "and", fontfile)
+      fonts[font].otf = fontfile
+    elif fontfile.startswith("./unhinted/variable-ttf"):
+      if fonts[font].vf and fonts[font].vf != fontfile and font != "":
+        print("duplicate vf", font, ":", fonts[font].vf, "and", fontfile)
+      fonts[font].vf = fontfile
+    elif fontfile.startswith("./unhinted/slim-variable-ttf"):
+      if fonts[font].slimvf and fonts[font].slimvf != fontfile and font != "":
+        print("duplicate slim", font, ":", fonts[font].slimvf, "and", fontfile)
+      fonts[font].slimvf = fontfile
+    else:
+      print("Unknown font directory: ", ', '.join(row))
+      raise NotImplementedError
+
+    fonts[font].extra = extrafontpattern.match(font) != None or font == ""
+
+# inconsistent names, see https://github.com/googlefonts/noto-fonts/issues/2317
+# for these names the VF names seem acceptable
+lightlightpattern = re.compile(r"Noto Sans Hebrew (New )?Light Light")
+hmongpattern = re.compile(r"Noto Serif Nyiakeng Puachue Hmong .+")
+
+# compute noto-font files
+
+mode = "vf" # "otf" "slimvf" "vf"
+modeextra = "vf" # "otf" "vf"
+
+# force-add Display VF's to file list, see above
+# https://github.com/googlefonts/noto-fonts/issues/2315
+filelist = set(
+  ["./unhinted/variable-ttf/NotoSansDisplay-Italic-VF.ttf",
+  "./unhinted/variable-ttf/NotoSansDisplay-VF.ttf",
+  "./unhinted/variable-ttf/NotoSerifDisplay-VF.ttf"])
+
+for name, font in fonts.items():
+  if font.extra:
+    continue
+
+  if font.vf and mode == "vf":
+    if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name):
+      print("missing otf:", name, ", ", font.vf)
+      continue
+    filelist.add(font.vf)
+  elif font.slimvf and mode == "slimvf":
+    if not font.otf and font.slimvf != "":
+      print("missing otf:", name, ", ", font.slimvf)
+      continue
+    filelist.add(font.slimvf)
+  elif font.otf:
+    filelist.add(font.otf)
+  elif font.vf:
+    continue
+  else:
+    print("weird font: ", name, ", ", font)
+    raise NotImplementedError
+
+with open('noto_fonts_list.txt', 'w') as f:
+    for item in filelist:
+        f.write("%s\n" % item)
+
+extra_list = set()
+
+for name, font in fonts.items():
+  if font.vf in filelist or font.slimvf in filelist or font.otf in filelist:
+    continue
+
+  if font.vf and modeextra != "otf":
+    if not font.otf and name != "" and not lightlightpattern.match(name) and not hmongpattern.match(name):
+      print("missing otf:", name, ", ", font.vf)
+      continue
+    extra_list.add(font.vf)
+  elif font.otf:
+    extra_list.add(font.otf)
+  elif font.vf:
+    continue
+  else:
+    print("weird font: ", name, ", ", font)
+    raise NotImplementedError
+
+with open('noto_extra_list.txt', 'w') as f:
+    for item in extra_list:
+        f.write("%s\n" % item)
+

pkgs/data/fonts/noto-fonts/default.nix

@@ -11,32 +11,44 @@
 , imagemagick
 , zopfli
 , buildPackages
+, fontconfig
+, python3
 }:
 
 let
-  mkNoto = { pname, weights }:
+  noto-pkg =
     stdenvNoCC.mkDerivation {
-      inherit pname;
-      version = "2020-01-23";
+      pname = "noto-fonts";
+      version = "unstable-2022-04-25";
 
       src = fetchFromGitHub {
         owner = "googlefonts";
         repo = "noto-fonts";
-        rev = "f4726a2ec36169abd02a6d8abe67c8ff0236f6d8";
-        sha256 = "0zc1r7zph62qmvzxqfflsprazjf6x1qnwc2ma27kyzh6v36gaykw";
+        rev = "a19de47f845dbd4c61b884c7ff90ce993555d05d";
+        sparseCheckout = ''
+          unhinted/otf
+          unhinted/variable-ttf
+        '';
+        hash = "sha256-7Jzo7402CiOcn2q++zbAzn2tP/y2x7QJuwjcP1NoyLQ=";
       };
 
+      analyzeScan = ./analyze_scan.py;
+
+      nativeBuildInputs = [ buildPackages.fontconfig.bin buildPackages.python3 ];
+
+      outputs = [ "out" "extra" ];
+
       installPhase = ''
-        # We copy in reverse preference order -- unhinted first, then
-        # hinted -- to get the "best" version of each font while
-        # maintaining maximum coverage.
-        #
-        # TODO: install OpenType, variable versions?
-        local out_ttf=$out/share/fonts/truetype/noto
-        install -m444 -Dt $out_ttf phaseIII_only/unhinted/ttf/*/*-${weights}.ttf
-        install -m444 -Dt $out_ttf phaseIII_only/hinted/ttf/*/*-${weights}.ttf
-        install -m444 -Dt $out_ttf unhinted/*/*-${weights}.ttf
-        install -m444 -Dt $out_ttf hinted/*/*-${weights}.ttf
+        fc-scan ./unhinted -f '"%{fullname[0]}","%{file}"\n' > scan.csv
+        python $analyzeScan
+        local out_ttf=$out/share/fonts/google-noto
+        while read p; do
+          install -m444 -Dt $out_ttf "$p"
+        done <noto_fonts_list.txt
+        local extra_ttf=$extra/share/fonts/google-noto
+        while read p; do
+          install -m444 -Dt $extra_ttf "$p"
+        done <noto_extra_list.txt
       '';
 
       meta = with lib; {
@@ -54,12 +66,18 @@ let
           Google’s goal is to see “no more tofu”.  Noto has multiple styles and
           weights, and freely available to all.
 
+          noto-fonts contains Regular, Bold, and Light weights and Italic style,
+          and noto-fonts-extra contains the rest.
+
           This package also includes the Arimo, Cousine, and Tinos fonts.
         '';
         license = licenses.ofl;
         platforms = platforms.all;
         maintainers = with maintainers; [ mathnerd314 emily ];
       };
+
+      passthru.tests.noto-fonts = nixosTests.noto-fonts;
+
     };
 
   mkNotoCJK = { typeface, version, rev, sha256 }:
@@ -103,15 +121,8 @@ let
 in
 
 {
-  noto-fonts = mkNoto {
-    pname = "noto-fonts";
-    weights = "{Regular,Bold,Light,Italic,BoldItalic,LightItalic}";
-  };
-
-  noto-fonts-extra = mkNoto {
-    pname = "noto-fonts-extra";
-    weights = "{Black,Condensed,Extra,Medium,Semi,Thin}*";
-  };
+  noto-fonts = noto-pkg.out;
+  noto-fonts-extra = noto-pkg.extra;
 
   noto-fonts-cjk-sans = mkNotoCJK {
     typeface = "Sans";
@@ -182,6 +193,8 @@ in
       platforms = platforms.all;
       maintainers = with maintainers; [ mathnerd314 sternenseemann ];
     };
+
+    passthru.tests.noto-fonts = nixosTests.noto-fonts;
   };
 
   noto-fonts-emoji-blob-bin =