Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Luarocks pyupdater followup #133296

Merged
merged 2 commits into from
Aug 12, 2021
Merged

Luarocks pyupdater followup #133296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ddad24
update-luarocks-packages: use pluginupdate.py
Jul 30, 2021
5e4ece6
luaPackages: update
Aug 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 53 additions & 58 deletions maintainers/scripts/pluginupdate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@
}

log = logging.getLogger()
log.addHandler(logging.StreamHandler())


def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
"""Retry calling the decorated function using an exponential backoff.
Expand Down Expand Up @@ -203,7 +201,6 @@ def __init__(
name: str,
root: Path,
get_plugins: str,
generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None],
default_in: Optional[Path] = None,
default_out: Optional[Path] = None,
deprecated: Optional[Path] = None,
Expand All @@ -213,7 +210,6 @@ def __init__(
self.name = name
self.root = root
self.get_plugins = get_plugins
self._generate_nix = generate_nix
self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
self.default_out = default_out or root.joinpath("generated.nix")
self.deprecated = deprecated or root.joinpath("deprecated.json")
Expand All @@ -226,9 +222,9 @@ def get_current_plugins(self):
def load_plugin_spec(self, plugin_file) -> List[PluginDesc]:
return load_plugin_spec(plugin_file)

def generate_nix(self, plugins, outfile):
def generate_nix(self, plugins, outfile: str):
'''Returns nothing for now, writes directly to outfile'''
self._generate_nix(plugins, outfile)
raise NotImplementedError()

def get_update(self, input_file: str, outfile: str, proc: int):
return get_update(input_file, outfile, proc, editor=self)
Expand All @@ -237,9 +233,58 @@ def get_update(self, input_file: str, outfile: str, proc: int):
def attr_path(self):
return self.name + "Plugins"

def get_drv_name(self, name: str):
return self.attr_path + "." + name

def rewrite_input(self, *args, **kwargs):
return rewrite_input(*args, **kwargs)

def create_parser(self):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {self.name} plugins"
f"By default from {self.default_in} to {self.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=self.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=self.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser



Expand Down Expand Up @@ -466,54 +511,6 @@ def rewrite_input(
with open(input_file, "w") as f:
f.writelines(lines)

# TODO move to Editor ?
def parse_args(editor: Editor):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {editor.name} plugins"
f"By default from {editor.default_in} to {editor.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {editor.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=editor.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=editor.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser.parse_args()


def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
repo.index.add([str(f.resolve()) for f in files])
Expand Down Expand Up @@ -547,12 +544,10 @@ def update() -> dict:
return update


def update_plugins(editor: Editor):
def update_plugins(editor: Editor, args):
"""The main entry function of this module. All input arguments are grouped in the `Editor`."""

args = parse_args(editor)
log.setLevel(LOG_LEVELS[args.debug])

log.info("Start updating plugins")
nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
update = editor.get_update(args.input_file, args.outfile, args.proc)
Expand Down Expand Up @@ -581,7 +576,7 @@ def update_plugins(editor: Editor):
if autocommit:
commit(
nixpkgs_repo,
"{editor.attr_path}.{name}: init at {version}".format(
"{editor.get_drv_name name}: init at {version}".format(
editor=editor.name, name=plugin.normalized_name, version=plugin.version
),
[args.outfile, args.input_file],
Expand Down
149 changes: 82 additions & 67 deletions maintainers/scripts/update-luarocks-packages
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,17 @@ from dataclasses import dataclass
import subprocess
import csv
import logging
import textwrap
from multiprocessing.dummy import Pool

from typing import List
from typing import List, Tuple
from pathlib import Path

LOG_LEVELS = {
logging.getLevelName(level): level for level in [
logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
}

log = logging.getLogger()
log.addHandler(logging.StreamHandler())

ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent
from pluginupdate import Editor, parse_args, update_plugins, PluginDesc, CleanEnvironment
from pluginupdate import Editor, update_plugins, PluginDesc, CleanEnvironment, LOG_LEVELS, Cache

PKG_LIST="maintainers/scripts/luarocks-packages.csv"
TMP_FILE="$(mktemp)"
Expand Down Expand Up @@ -67,12 +64,11 @@ class LuaEditor(Editor):
def get_current_plugins(self):
return []

def load_plugin_spec(self, input_file) -> List[PluginDesc]:
def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
luaPackages = []
csvfilename=input_file
log.info("Loading package descriptions from %s", csvfilename)


with open(csvfilename, newline='') as csvfile:
reader = csv.DictReader(csvfile,)
for row in reader:
Expand All @@ -81,96 +77,115 @@ class LuaEditor(Editor):
luaPackages.append(plugin)
return luaPackages

def generate_nix(
self,
results: List[Tuple[LuaPlugin, str]],
outfilename: str
):

with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
header2 = textwrap.dedent(
# header2 = inspect.cleandoc(
"""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
f.write(header2)
for (plugin, nix_expr) in results:
f.write(f"{plugin.normalized_name} = {nix_expr}")
f.write(FOOTER)
f.flush()

# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)

print(f"updated {outfilename}")

@property
def attr_path(self):
return "luaPackages"

def get_update(self, input_file: str, outfile: str, _: int):
def get_update(self, input_file: str, outfile: str, proc: int):
cache: Cache = Cache(self.cache_file)
_prefetch = generate_pkg_nix

def update() -> dict:
plugin_specs = self.load_plugin_spec(input_file)
sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())

try:
pool = Pool(processes=proc)
results = pool.map(_prefetch, sorted_plugin_specs)
finally:
pass

self.generate_nix(plugin_specs, outfile)
self.generate_nix(results, outfile)

redirects = []
return redirects

return update

def rewrite_input(self, *args, **kwargs):
def rewrite_input(self, input_file: str, *args, **kwargs):
# vim plugin reads the file before update but that shouldn't be our case
# not implemented yet
# fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
# input_file = "toto.csv"
# with open(input_file, newline='') as csvfile:
# writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# writer.writeheader()
# for row in reader:
# # name,server,version,luaversion,maintainers
# plugin = LuaPlugin(**row)
# luaPackages.append(plugin)
pass

def generate_nix(
plugins: List[LuaPlugin],
outfilename: str
):
sorted_plugins = sorted(plugins, key=lambda v: v.name.lower())

# plug = {}
# selon le manifest luarocks.org/manifest
def _generate_pkg_nix(plug):
cmd = [ "luarocks", "nix", plug.name]
if plug.server:
cmd.append(f"--only-server={plug.server}")

if plug.maintainers:
cmd.append(f"--maintainers={plug.maintainers}")

if plug.version:
cmd.append(plug.version)

if plug.luaversion:
with CleanEnvironment():
local_pkgs = str(ROOT.resolve())
cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]

log.debug("running %s", cmd2)
lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
cmd.append(f"--lua-dir={lua_drv_path}/bin")

log.debug("running %s", cmd)
output = subprocess.check_output(cmd, text=True)
return output

with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
f.write("""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
def generate_pkg_nix(plug: LuaPlugin):
'''
Generate nix expression for a luarocks package
Our cache key associates "p.name-p.version" to its rockspec
'''
log.debug("Generating nix expression for %s", plug.name)
cmd = [ "luarocks", "nix", plug.name]

for plugin in sorted_plugins:
if plug.server:
cmd.append(f"--only-server={plug.server}")

nix_expr = _generate_pkg_nix(plugin)
f.write(f"{plugin.normalized_name} = {nix_expr}"
)
f.write(FOOTER)
f.flush()
if plug.maintainers:
cmd.append(f"--maintainers={plug.maintainers}")

# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)
if plug.version:
cmd.append(plug.version)

print(f"updated {outfilename}")
if plug.luaversion:
with CleanEnvironment():
local_pkgs = str(ROOT.resolve())
cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]

def load_plugin_spec():
pass
log.debug("running %s", ' '.join(cmd2))
lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
cmd.append(f"--lua-dir={lua_drv_path}/bin")

log.debug("running %s", cmd)
output = subprocess.check_output(cmd, text=True)
return (plug, output)

def main():

editor = LuaEditor("lua", ROOT, '', generate_nix,
editor = LuaEditor("lua", ROOT, '',
default_in = ROOT.joinpath(PKG_LIST),
default_out = ROOT.joinpath(GENERATED_NIXFILE)
)

args = parse_args(editor)
parser = editor.create_parser()
args = parser.parse_args()
log.setLevel(LOG_LEVELS[args.debug])

update_plugins(editor)
update_plugins(editor, args)


if __name__ == "__main__":
Expand Down
Loading