Add support for SCIP + renaming. (#742)

* Document why the flags are being modified after-the-fact. This kind of nonsense is why I hate global variables. Yuck. * Add support for SCIP uploads.
sourcegraph · May 13, 2022 · bac5a37 · bac5a37
1 parent ee83879
commit bac5a37
Show file tree

Hide file tree

Showing 4 changed files with 261 additions and 122 deletions.
diff --git a/cmd/src/lsif_upload_flags.go b/cmd/src/lsif_upload_flags.go
@@ -4,14 +4,16 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 	"github.com/sourcegraph/sourcegraph/lib/output"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/sourcegraph/scip/bindings/go/scip"
 	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/reader"
-	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsiftyped"
 	"github.com/sourcegraph/sourcegraph/lib/codeintel/upload"
 
 	"github.com/sourcegraph/src-cli/internal/api"
@@ -111,7 +113,7 @@ func parseAndValidateLSIFUploadFlags(args []string) (*output.Output, error) {
 		return nil, err
 	}
 
-	if err := handleLSIFTyped(out); err != nil {
+	if err := handleSCIP(out); err != nil {
 		return nil, err
 	}
 
@@ -155,35 +157,68 @@ type argumentInferenceError struct {
 	err      error
 }
 
-func handleLSIFTyped(out *output.Output) error {
-	if strings.HasSuffix(lsifUploadFlags.file, ".lsif-typed") {
-		// The user explicitly passed in a -file flag that points to an LSIF Typed index.
-		inputFile := lsifUploadFlags.file
-		outputFile := strings.TrimSuffix(inputFile, "-typed")
+func replaceExtension(oldPath string, newExtension string) string {
+	oldExtLen := len(path.Ext(oldPath))
+	if oldExtLen == 0 {
+		panic(fmt.Sprintf("Expected path %s to have an extension", oldPath))
+	}
+	return oldPath[:len(oldPath)-oldExtLen] + newExtension
+}
+
+func replaceBaseName(oldPath string, newBaseName string) string {
+	if filepath.Dir(newBaseName) != "." {
+		panic(fmt.Sprintf("Expected bare file name but found %s", newBaseName))
+	}
+	return filepath.Join(filepath.Dir(oldPath), newBaseName)
+}
+
+func handleSCIP(out *output.Output) error {
+	fileExt := path.Ext(lsifUploadFlags.file)
+	if len(fileExt) == 0 {
+		return errors.Newf("missing file extension for %s; expected .scip", lsifUploadFlags.file)
+	}
+	inputFile := lsifUploadFlags.file
+	if fileExt == ".scip" || fileExt == ".lsif-typed" {
+		// The user explicitly passed in a -file flag that points to an SCIP index.
+		outputFile := replaceExtension(inputFile, ".lsif")
+		if filepath.Base(inputFile) == "index.scip" {
+			outputFile = replaceBaseName(inputFile, "dump.lsif")
+		}
+		// HACK: Modify the flags to point to the output file, because
+		// that field of the flags is read when performing the upload.
 		lsifUploadFlags.file = outputFile
-		return convertLSIFTypedToLSIFGraph(out, inputFile, outputFile)
+		return convertSCIPToLSIFGraph(out, inputFile, outputFile)
 	}
 
-	if _, err := os.Stat(lsifUploadFlags.file); err == nil {
+	if _, err := os.Stat(inputFile); err == nil {
 		// Do nothing, the provided -flag flag points to an existing
-		// file that does not have the file extension `*.lsif-typed`.
+		// file that does not have the file extension `.lsif-typed` or `.scip`.
 		return nil
 	}
 
-	lsifTypedFile := lsifUploadFlags.file + "-typed"
-	if _, err := os.Stat(lsifTypedFile); os.IsNotExist(err) {
-		// The inferred path of the sibling `*.lsif-typed` file does not exist.
-		return nil
+	scipFile := replaceExtension(inputFile, ".scip")
+	if _, err := os.Stat(scipFile); os.IsNotExist(err) {
+		// The input may be named 'dump.lsif', but the default name for SCIP
+		// indexes is 'index.scip', not 'dump.scip'.
+		scipFile = replaceBaseName(inputFile, "index.scip")
+		if _, err := os.Stat(scipFile); os.IsNotExist(err) {
+			lsifTypedFile := replaceExtension(inputFile, ".lsif-typed")
+			if _, err := os.Stat(lsifTypedFile); os.IsNotExist(err) {
+				// There is no `*.scip` or `*.lsif-typed` file for the inferred path.
+				return nil
+			}
+			scipFile = lsifTypedFile
+		}
 	}
 
 	// The provided -file flag points to an `*.lsif` file that doesn't exist
-	// so we convert the sibling `*.lsif-typed` file (which we confirmed exists).
-	return convertLSIFTypedToLSIFGraph(out, lsifTypedFile, lsifUploadFlags.file)
+	// so we convert the sibling file (which we confirmed exists).
+	return convertSCIPToLSIFGraph(out, scipFile, lsifUploadFlags.file)
 }
 
-// Reads the LSIF Typed encoded input file and writes the corresponding LSIF
+// Reads the SCIP encoded input file and writes the corresponding LSIF
 // Graph encoded output file.
-func convertLSIFTypedToLSIFGraph(out *output.Output, inputFile, outputFile string) error {
+func convertSCIPToLSIFGraph(out *output.Output, inputFile, outputFile string) error {
 	if out != nil {
 		out.Writef("%s  Converting %s into %s", output.EmojiInfo, inputFile, outputFile)
 	}
@@ -197,12 +232,12 @@ func convertLSIFTypedToLSIFGraph(out *output.Output, inputFile, outputFile strin
 	if err != nil {
 		panic(err)
 	}
-	index := lsiftyped.Index{}
+	index := scip.Index{}
 	err = proto.Unmarshal(data, &index)
 	if err != nil {
 		panic(errors.Wrapf(err, "failed to parse protobuf file '%s'", inputFile))
 	}
-	els, err := reader.ConvertTypedIndexToGraphIndex(&index)
+	els, err := scip.ConvertSCIPToLSIF(&index)
 	if err != nil {
 		panic(errors.Wrapf(err, "failed reader.ConvertTypedIndexToGraphIndex"))
 	}

diff --git a/cmd/src/lsif_upload_flags_test.go b/cmd/src/lsif_upload_flags_test.go
@@ -5,73 +5,94 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsiftyped"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/proto"
+
+	"github.com/sourcegraph/scip/bindings/go/scip"
 )
 
-var exampleLsifTypedIndex = lsiftyped.Index{
-	Metadata: &lsiftyped.Metadata{
-		TextDocumentEncoding: lsiftyped.TextEncoding_UTF8,
-		ToolInfo: &lsiftyped.ToolInfo{
+var exampleSCIPIndex = scip.Index{
+	Metadata: &scip.Metadata{
+		TextDocumentEncoding: scip.TextEncoding_UTF8,
+		ToolInfo: &scip.ToolInfo{
 			Name:    "hello",
 			Version: "1.0.0",
 		},
 	},
 }
 
-var exampleLsifGraphString = `{"id":1,"version":"0.4.3","positionEncoding":"utf-8","toolInfo":{"name":"hello","version":"1.0.0"},"type":"vertex","label":"metaData"}
+var exampleLSIFString = `{"id":1,"version":"0.4.3","positionEncoding":"utf-8","toolInfo":{"name":"hello","version":"1.0.0"},"type":"vertex","label":"metaData"}
 `
 
-func exampleLsifTypedBytes(t *testing.T) []byte {
-	bytes, err := proto.Marshal(&exampleLsifTypedIndex)
+func exampleSCIPBytes(t *testing.T) []byte {
+	bytes, err := proto.Marshal(&exampleSCIPIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
 	return bytes
 }
 
-func createTempLsifTypedFile(t *testing.T) (typedFile, graphFile string) {
+func createTempSCIPFile(t *testing.T, scipFileName string) (scipFilePath string, lsifFilePath string) {
 	dir := t.TempDir()
-	typedFile = filepath.Join(dir, "dump.lsif-typed")
-	graphFile = filepath.Join(dir, "dump.lsif")
-	err := os.WriteFile(typedFile, exampleLsifTypedBytes(t), 0755)
+	require.NotEqual(t, "", scipFileName)
+	scipFilePath = filepath.Join(dir, scipFileName)
+	lsifFilePath = filepath.Join(dir, "dump.lsif")
+	err := os.WriteFile(scipFilePath, exampleSCIPBytes(t), 0755)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return typedFile, graphFile
+	return scipFilePath, lsifFilePath
 }
 
-func assertLsifGraphOutput(t *testing.T, lsifGraphFile, expectedGraphString string) {
+func assertLSIFOutput(t *testing.T, lsifFile, expectedLSIFString string) {
 	out := lsifUploadOutput()
-	handleLSIFTyped(out)
-	lsifGraph, err := os.ReadFile(lsifGraphFile)
+	handleSCIP(out)
+	lsif, err := os.ReadFile(lsifFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	obtained := string(lsifGraph)
-	if obtained != expectedGraphString {
+	obtained := string(lsif)
+	if obtained != expectedLSIFString {
 		t.Fatalf("unexpected LSIF output %s", obtained)
 	}
-	if lsifGraphFile != lsifUploadFlags.file {
-		t.Fatalf("unexpected lsifUploadFlag.file value %s, expected %s", lsifUploadFlags.file, lsifGraphFile)
+	if lsifFile != lsifUploadFlags.file {
+		t.Fatalf("unexpected lsifUploadFlag.file value %s, expected %s", lsifUploadFlags.file, lsifFile)
 	}
 }
 
-func TestImplicitlyConvertLsifTypedIntoGraph(t *testing.T) {
-	_, graphFile := createTempLsifTypedFile(t)
-	lsifUploadFlags.file = graphFile
-	assertLsifGraphOutput(t, graphFile, exampleLsifGraphString)
+func TestImplicitlyConvertSCIPIntoLSIF(t *testing.T) {
+	for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} {
+		_, lsifFile := createTempSCIPFile(t, filename)
+		lsifUploadFlags.file = lsifFile
+		assertLSIFOutput(t, lsifFile, exampleLSIFString)
+	}
 }
 
-func TestImplicitlyIgnoreLsifTyped(t *testing.T) {
-	_, graphFile := createTempLsifTypedFile(t)
-	lsifUploadFlags.file = graphFile
-	os.WriteFile(graphFile, []byte("hello world"), 0755)
-	assertLsifGraphOutput(t, graphFile, "hello world")
+func TestImplicitlyIgnoreSCIP(t *testing.T) {
+	for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} {
+		_, lsifFile := createTempSCIPFile(t, filename)
+		lsifUploadFlags.file = lsifFile
+		os.WriteFile(lsifFile, []byte("hello world"), 0755)
+		assertLSIFOutput(t, lsifFile, "hello world")
+	}
 }
 
-func TestExplicitlyConvertLsifTypedIntoGraph(t *testing.T) {
-	typedFile, graphFile := createTempLsifTypedFile(t)
-	lsifUploadFlags.file = typedFile
-	assertLsifGraphOutput(t, graphFile, exampleLsifGraphString)
+func TestExplicitlyConvertSCIPIntoGraph(t *testing.T) {
+	for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} {
+		scipFile, lsifFile := createTempSCIPFile(t, filename)
+		lsifUploadFlags.file = scipFile
+		assertLSIFOutput(t, lsifFile, exampleLSIFString)
+	}
+}
+
+func TestReplaceExtension(t *testing.T) {
+	require.Panics(t, func() { replaceExtension("foo", ".xyz") })
+	require.Equal(t, "foo.xyz", replaceExtension("foo.abc", ".xyz"))
+}
+
+func TestReplaceBaseName(t *testing.T) {
+	require.Panics(t, func() { replaceBaseName("mydir", filepath.Join("dir", "file")) })
+
+	require.Equal(t, filepath.Join("a", "d.e"),
+		replaceBaseName(filepath.Join("a", "b.c"), "d.e"))
 }
diff --git a/go.mod b/go.mod
@@ -16,30 +16,47 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/mattn/go-isatty v0.0.14
 	github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c
-	github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/sourcegraph/go-diff v0.6.1
 	github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
-	github.com/sourcegraph/sourcegraph/lib v0.0.0-20220506010657-06e02488a3d7
+	github.com/sourcegraph/scip v0.0.0-20220513191902-a9bebaa57565
+	github.com/sourcegraph/sourcegraph/lib v0.0.0-20220511160847-5a43d3ea24eb
 	github.com/stretchr/testify v1.7.1
-	golang.org/x/net v0.0.0-20220325170049-de3da57026de
+	golang.org/x/net v0.0.0-20220412020605-290c469a71a5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/protobuf v1.28.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Masterminds/sprig v2.15.0+incompatible // indirect
+	github.com/aokoli/goutils v1.0.1 // indirect
+	github.com/bufbuild/buf v1.4.0 // indirect
 	github.com/cockroachdb/errors v1.8.9 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect
 	github.com/getsentry/sentry-go v0.12.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/huandu/xstrings v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.4 // indirect
 	github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
+	github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect
+	github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -48,17 +65,35 @@ require (
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/profile v1.6.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pseudomuto/protoc-gen-doc v1.5.1 // indirect
+	github.com/pseudomuto/protokit v0.2.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+	golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect
+	google.golang.org/grpc v1.45.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )