cue/load: minimal module fetching implementation

This is just a proof of concept for now - the details will probably
change, but it demonstrates MVS module version resolution and
fetching modules from a server.

Note that public behaviour does not change here - the new behavior
is only triggered by setting the `load.Config.Registry` field to a non-empty
URL.

The `Config.absDirFromImportPath` method became `loader.absDirFromImportPath`
so was moved - the only changes are to use `l.cfg` instead of `c` and the
`if l.cfg.Registry` conditional code.

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: I7c5e0888db780f23a6d58081e70680a48dce509f
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/549762
Reviewed-by: Marcel van Lohuizen <mpvl@gmail.com>
Unity-Result: CUEcueckoo <cueckoo@cuelang.org>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>

cue/load/config.go

@@ -15,7 +15,9 @@
 package load
 
 import (
+	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -276,7 +278,11 @@ type Config struct {
 	// the corresponding build.File will be associated with the full buffer.
 	Stdin io.Reader
 
-	fileSystem
+	// Registry holds the URL of the CUE registry. If it has no scheme, https:// is assumed
+	// as a prefix. THIS IS EXPERIMENTAL FOR NOW. DO NOT USE.
+	Registry string
+
+	fileSystem fileSystem
 }
 
 func (c *Config) stdin() io.Reader {
@@ -315,53 +321,6 @@ func addImportQualifier(pkg importPath, name string) (importPath, errors.Error)
 	return pkg, nil
 }
 
-// absDirFromImportPath converts a giving import path to an absolute directory
-// and a package name. The root directory must be set.
-//
-// The returned directory may not exist.
-func (c *Config) absDirFromImportPath(pos token.Pos, p importPath) (absDir, name string, err errors.Error) {
-	if c.ModuleRoot == "" {
-		return "", "", errors.Newf(pos, "cannot import %q (root undefined)", p)
-	}
-
-	// Extract the package name.
-
-	name = string(p)
-	switch i := strings.LastIndexAny(name, "/:"); {
-	case i < 0:
-	case p[i] == ':':
-		name = string(p[i+1:])
-		p = p[:i]
-
-	default: // p[i] == '/'
-		name = string(p[i+1:])
-	}
-
-	// TODO: fully test that name is a valid identifier.
-	if name == "" {
-		err = errors.Newf(pos, "empty package name in import path %q", p)
-	} else if strings.IndexByte(name, '.') >= 0 {
-		err = errors.Newf(pos,
-			"cannot determine package name for %q (set explicitly with ':')", p)
-	}
-
-	// Determine the directory.
-
-	sub := filepath.FromSlash(string(p))
-	switch hasPrefix := strings.HasPrefix(string(p), c.Module); {
-	case hasPrefix && len(sub) == len(c.Module):
-		absDir = c.ModuleRoot
-
-	case hasPrefix && p[len(c.Module)] == '/':
-		absDir = filepath.Join(c.ModuleRoot, sub[len(c.Module)+1:])
-
-	default:
-		absDir = filepath.Join(GenPath(c.ModuleRoot), sub)
-	}
-
-	return absDir, name, err
-}
-
 // Complete updates the configuration information. After calling complete,
 // the following invariants hold:
 //   - c.Dir is an absolute path.
@@ -408,6 +367,16 @@ func (c Config) complete() (cfg *Config, err error) {
 	} else if !filepath.IsAbs(c.ModuleRoot) {
 		c.ModuleRoot = filepath.Join(c.Dir, c.ModuleRoot)
 	}
+	if c.Registry != "" {
+		u, err := url.Parse(c.Registry)
+		if err != nil {
+			return nil, fmt.Errorf("invalid registry URL %q: %v", c.Registry, err)
+		}
+		if u.Scheme == "" {
+			u.Scheme = "https"
+			c.Registry = u.String()
+		}
+	}
 	if err := c.loadModule(); err != nil {
 		return nil, err
 	}

cue/load/import.go

@@ -69,11 +69,6 @@ func (l *loader) importPkg(pos token.Pos, p *build.Instance) []*build.Instance {
 		return []*build.Instance{p}
 	}
 
-	if !strings.HasPrefix(p.Dir, cfg.ModuleRoot) {
-		err := errors.Newf(token.NoPos, "module root not defined", p.DisplayPath)
-		return retErr(err)
-	}
-
 	fp := newFileProcessor(cfg, p, l.tagger)
 
 	if p.PkgName == "" {
@@ -322,7 +317,7 @@ func (l *loader) importPathFromAbsDir(absDir fsPath, key string) (importPath, er
 }
 
 func (l *loader) newInstance(pos token.Pos, p importPath) *build.Instance {
-	dir, name, err := l.cfg.absDirFromImportPath(pos, p)
+	dir, name, err := l.absDirFromImportPath(pos, p)
 	i := l.cfg.Context.NewInstance(dir, l.loadFunc)
 	i.Dir = dir
 	i.PkgName = name
@@ -334,3 +329,73 @@ func (l *loader) newInstance(pos token.Pos, p importPath) *build.Instance {
 
 	return i
 }
+
+// absDirFromImportPath converts a giving import path to an absolute directory
+// and a package name. The root directory must be set.
+//
+// The returned directory may not exist.
+func (l *loader) absDirFromImportPath(pos token.Pos, p importPath) (absDir, name string, err errors.Error) {
+	if l.cfg.ModuleRoot == "" {
+		return "", "", errors.Newf(pos, "cannot import %q (root undefined)", p)
+	}
+
+	// Extract the package name.
+
+	name = string(p)
+	switch i := strings.LastIndexAny(name, "/:"); {
+	case i < 0:
+	case p[i] == ':':
+		name = string(p[i+1:])
+		p = p[:i]
+
+	default: // p[i] == '/'
+		name = string(p[i+1:])
+	}
+
+	// TODO: fully test that name is a valid identifier.
+	if name == "" {
+		err = errors.Newf(pos, "empty package name in import path %q", p)
+	} else if strings.IndexByte(name, '.') >= 0 {
+		err = errors.Newf(pos,
+			"cannot determine package name for %q (set explicitly with ':')", p)
+	}
+
+	// Determine the directory.
+
+	sub := filepath.FromSlash(string(p))
+	switch hasPrefix := strings.HasPrefix(string(p), l.cfg.Module); {
+	case hasPrefix && len(sub) == len(l.cfg.Module):
+		absDir = l.cfg.ModuleRoot
+
+	case hasPrefix && p[len(l.cfg.Module)] == '/':
+		absDir = filepath.Join(l.cfg.ModuleRoot, sub[len(l.cfg.Module)+1:])
+
+	default:
+		// TODO predicate registry-aware lookup on module.cue-declared CUE version?
+		if l.cfg.Registry != "" {
+			var err error
+			absDir, err = l.externalPackageDir(p)
+			if err != nil {
+				// TODO why can't we use %w ?
+				return "", name, errors.Newf(token.NoPos, "cannot get directory for external module %q: %v", p, err)
+			}
+		} else {
+			absDir = filepath.Join(GenPath(l.cfg.ModuleRoot), sub)
+		}
+	}
+
+	return absDir, name, err
+}
+
+func (l *loader) externalPackageDir(p importPath) (dir string, err error) {
+	m, subPath, ok := l.deps.lookup(p)
+	if !ok {
+		return "", fmt.Errorf("no dependency found for import path %q", p)
+	}
+
+	dir, err = l.regClient.getModContents(m)
+	if err != nil {
+		return "", fmt.Errorf("cannot get contents for %v: %v", m, err)
+	}
+	return filepath.Join(dir, filepath.FromSlash(subPath)), nil
+}

cue/load/instances.go

@@ -20,6 +20,8 @@ package load
 //    - go/build
 
 import (
+	"os"
+
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/internal/filetypes"
@@ -45,8 +47,24 @@ func Instances(args []string, c *Config) []*build.Instance {
 		return []*build.Instance{c.newErrInstance(err)}
 	}
 	c = newC
+	// TODO use predictable location
+	var deps *dependencies
+	var regClient *registryClient
+	if c.Registry != "" {
+		// TODO use configured cache directory.
+		tmpDir, err := os.MkdirTemp("", "cue-load-")
+		if err != nil {
+			return []*build.Instance{c.newErrInstance(err)}
+		}
+		regClient = newRegistryClient(c.Registry, tmpDir)
+		deps1, err := resolveDependencies(c.modFile, regClient)
+		if err != nil {
+			return []*build.Instance{c.newErrInstance(err)}
+		}
+		deps = deps1
+	}
 	tg := newTagger(c)
-	l := newLoader(c, tg)
+	l := newLoader(c, tg, deps, regClient)
 
 	if c.Context == nil {
 		c.Context = build.NewContext(