Skip to content

Commit

Permalink
Merge cockroachdb#68412
Browse files Browse the repository at this point in the history 
68412: ui: use yarn workspaces for project structure r=koorosh a=koorosh

Note. This is a huge PR with lots of changes which mostly affect files location.

The purpose of this change
This is a part of build improvements that include optimization and speed up of builds on CI to get fast response on code changes.

The main changes introduced with this change:
- added top level `pkg/ui/package.json` that defines yarn workspaces and includes project wide dependencies required for all packages (including for building protobufs).
- `cluster-ui` NPM package is moved to `pkg/ui/workspaces/cluster-ui`
- main `ui` package is moved to `pkg/ui/workspaces/db-console`
- Make file configuration adjusted to run builds with new project structure
- typescript version is changed to 4.x for all packages to have consistency across of projects. This also affected linting rules which now is more strict and required fix loose typings.

Note for reviewers. It is easier to review on per commit basis (to distinguish lint autofix changes etc)

Release note: none

Release justification: non-production code changes


Co-authored-by: Andrii Vorobiov <and.vorobiov@gmail.com>
  • Loading branch information
@koorosh
craig[bot] and koorosh committed Aug 25, 2021
2 parents 967ed00 + 2d0d15f commit 1b47ae3
Show file tree
Hide file tree
Showing 1,059 changed files with 12,372 additions and 25,950 deletions.
121 changes: 49 additions & 72 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,34 +355,20 @@ $(GITHOOKSDIR)/%: githooks/%
@ln -s ../../$(basename $<) $(dir $@)
endif

.SECONDARY: protobufjs-cli-fix-deps
protobufjs-cli-fix-deps:
# Prevent ProtobufJS from trying to install its own packages because a) the
# the feature is buggy, and b) it introduces an unnecessary dependency on NPM.
# See: https://github.com/dcodeIO/protobuf.js/issues/716.
# We additionally pin the dependencies by linking in a lock file for
# reproducable builds.
$(NODE_RUN) pkg/ui/bin/gen-protobuf-cli-deps.js > pkg/ui/node_modules/protobufjs/cli/package.json
ln -sf ../../../yarn.protobufjs-cli.lock pkg/ui/node_modules/protobufjs/cli/yarn.lock
$(NODE_RUN) -C pkg/ui/node_modules/protobufjs/cli yarn install --offline

.SECONDARY: pkg/ui/yarn.cluster-ui.installed
pkg/ui/yarn.cluster-ui.installed: pkg/ui/cluster-ui/package.json pkg/ui/cluster-ui/yarn.lock pkg/ui/src/js/protos.js pkg/ui/src/js/protos.d.ts | bin/.submodules-initialized
$(NODE_RUN) -C pkg/ui/cluster-ui yarn install --offline
# This ensures that any update to the protobuf will be picked up by cluster-ui.
$(NODE_RUN) -C pkg/ui/cluster-ui yarn upgrade file:pkg/ui/src/js
touch $@

.SECONDARY: pkg/ui/yarn.protobuf.installed
pkg/ui/yarn.protobuf.installed: pkg/ui/src/js/package.json pkg/ui/src/js/yarn.lock pkg/ui/yarn.protobufjs-cli.lock | bin/.submodules-initialized
$(NODE_RUN) -C pkg/ui/src/js yarn install --offline
$(MAKE) protobufjs-cli-fix-deps
touch $@
CLUSTER_UI_JS := pkg/ui/cluster-ui/dist/main.js

.SECONDARY: $(CLUSTER_UI_JS)
$(CLUSTER_UI_JS): $(shell find pkg/ui/workspaces/cluster-ui/src -type f | sed 's/ /\\ /g') pkg/ui/yarn.installed pkg/ui/workspaces/db-console/src/js/protos.d.ts | bin/.submodules-initialized
$(NODE_RUN) -C pkg/ui/workspaces/cluster-ui yarn build

.SECONDARY: pkg/ui/yarn.installed
pkg/ui/yarn.installed: pkg/ui/package.json pkg/ui/yarn.lock pkg/ui/yarn.cluster-ui.installed | bin/.submodules-initialized
$(NODE_RUN) -C pkg/ui yarn install --offline
$(MAKE) protobufjs-cli-fix-deps
pkg/ui/yarn.installed: pkg/ui/package.json pkg/ui/yarn.lock | bin/.submodules-initialized
@# Do not install optional dependencies as far as they are required for UI development only
@# and should not be installed for production builds.
@# Also some of linux distributives (that are used as development env) don't support some of
@# optional dependencies (i.e. cypress) so it is important to make these deps optional.
$(NODE_RUN) -C pkg/ui yarn install --ignore-optional
@# We remove this broken dependency again in pkg/ui/webpack.config.js.
@# See the comment there for details.
rm -rf pkg/ui/node_modules/@types/node
Expand Down Expand Up @@ -1270,12 +1256,12 @@ PBTS := $(NODE_RUN) pkg/ui/node_modules/.bin/pbts
# JavaScript only needs the entrypoint protobufs to be listed. It automatically
# compiles any protobufs the entrypoints depend upon.
JS_PROTOS_CCL := $(filter %/ccl/storageccl/engineccl/enginepbccl/stats.proto,$(GO_PROTOS))
UI_JS_CCL := pkg/ui/ccl/src/js/protos.js
UI_TS_CCL := pkg/ui/ccl/src/js/protos.d.ts
UI_JS_CCL := pkg/ui/workspaces/db-console/ccl/src/js/protos.js
UI_TS_CCL := pkg/ui/workspaces/db-console/ccl/src/js/protos.d.ts
UI_PROTOS_CCL := $(UI_JS_CCL) $(UI_TS_CCL)

UI_JS_OSS := pkg/ui/src/js/protos.js
UI_TS_OSS := pkg/ui/src/js/protos.d.ts
UI_JS_OSS := pkg/ui/workspaces/db-console/src/js/protos.js
UI_TS_OSS := pkg/ui/workspaces/db-console/src/js/protos.d.ts
UI_PROTOS_OSS := $(UI_JS_OSS) $(UI_TS_OSS)

$(GOGOPROTO_PROTO): bin/.submodules-initialized
Expand Down Expand Up @@ -1304,22 +1290,22 @@ bin/.gw_protobuf_sources: $(GW_SERVER_PROTOS) $(GW_TS_PROTOS) $(GO_PROTOS) $(GOG
# typescript definitions for the proto files afterwards.

.SECONDARY: $(UI_JS_CCL)
$(UI_JS_CCL): $(GW_PROTOS) $(GO_PROTOS) $(JS_PROTOS_CCL) pkg/ui/yarn.protobuf.installed | bin/.submodules-initialized
$(UI_JS_CCL): $(GW_PROTOS) $(GO_PROTOS) $(JS_PROTOS_CCL) pkg/ui/yarn.installed | bin/.submodules-initialized
# Add comment recognized by reviewable.
echo '// GENERATED FILE DO NOT EDIT' > $@
$(PBJS) -t static-module -w es6 --strict-long --keep-case --path pkg --path ./vendor/github.com --path $(GOGO_PROTOBUF_PATH) --path $(ERRORS_PATH) --path $(COREOS_PATH) --path $(PROMETHEUS_PATH) --path $(GRPC_GATEWAY_GOOGLEAPIS_PATH) $(filter %.proto,$(GW_PROTOS) $(JS_PROTOS_CCL)) >> $@

.SECONDARY: $(UI_JS_OSS)
$(UI_JS_OSS): $(GW_PROTOS) $(GO_PROTOS) pkg/ui/yarn.protobuf.installed | bin/.submodules-initialized
$(UI_JS_OSS): $(GW_PROTOS) $(GO_PROTOS) pkg/ui/yarn.installed | bin/.submodules-initialized
# Add comment recognized by reviewable.
echo '// GENERATED FILE DO NOT EDIT' > $@
$(PBJS) -t static-module -w es6 --strict-long --keep-case --path pkg --path ./vendor/github.com --path $(GOGO_PROTOBUF_PATH) --path $(ERRORS_PATH) --path $(COREOS_PATH) --path $(PROMETHEUS_PATH) --path $(GRPC_GATEWAY_GOOGLEAPIS_PATH) $(filter %.proto,$(GW_PROTOS)) >> $@

# End of PBJS-generated files.

.SECONDARY: $(UI_TS_CCL) $(UI_TS_OSS)
$(UI_TS_CCL): $(UI_JS_CCL) pkg/ui/yarn.protobuf.installed
$(UI_TS_OSS): $(UI_JS_OSS) pkg/ui/yarn.protobuf.installed
$(UI_TS_CCL): $(UI_JS_CCL) pkg/ui/yarn.installed
$(UI_TS_OSS): $(UI_JS_OSS) pkg/ui/yarn.installed
$(UI_TS_CCL) $(UI_TS_OSS):
# Add comment recognized by reviewable.
echo '// GENERATED FILE DO NOT EDIT' > $@
Expand All @@ -1337,33 +1323,24 @@ ui-generate: pkg/ui/distccl/bindata.go

.PHONY: ui-fonts
ui-fonts:
pkg/ui/scripts/font-gen
pkg/ui/workspaces/db-console/scripts/font-gen

.PHONY: ui-topo
ui-topo: pkg/ui/yarn.installed
pkg/ui/scripts/topo.js
pkg/ui/workspaces/db-console/scripts/topo.js

.PHONY: ui-lint
ui-lint: pkg/ui/yarn.installed $(UI_PROTOS_OSS) $(UI_PROTOS_CCL)
$(NODE_RUN) -C pkg/ui $(STYLINT) -c .stylintrc styl
$(NODE_RUN) -C pkg/ui $(TSC)
$(NODE_RUN) -C pkg/ui yarn lint
@if $(NODE_RUN) -C pkg/ui yarn list | grep phantomjs; then echo ^ forbidden UI dependency >&2; exit 1; fi
$(NODE_RUN) -C pkg/ui/cluster-ui yarn --cwd pkg/ui/cluster-ui lint

CLUSTER_UI_JS := pkg/ui/cluster-ui/dist/main.js

.SECONDARY: $(CLUSTER_UI_JS)

# The grep -v " " is to exclude a couple of font files that have spaces in their filenames, which
# Make is unable to deal with as far as I can tell.
$(CLUSTER_UI_JS): pkg/ui/yarn.installed $(shell find pkg/ui/cluster-ui/src -type f | grep -v " ")
$(NODE_RUN) -C pkg/ui/cluster-ui yarn build
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(STYLINT) -c .stylintrc styl
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(TSC)
$(NODE_RUN) -C pkg/ui/workspaces/db-console yarn lint
@if $(NODE_RUN) -C pkg/ui/workspaces/db-console yarn list | grep phantomjs; then echo ^ forbidden UI dependency >&2; exit 1; fi
$(NODE_RUN) -C pkg/ui/workspaces/cluster-ui yarn --cwd pkg/ui/workspaces/cluster-ui lint

# DLLs are Webpack bundles, not Windows shared libraries. See "DLLs for speedy
# builds" in the UI README for details.
UI_CCL_DLLS := pkg/ui/dist/protos.ccl.dll.js pkg/ui/dist/vendor.oss.dll.js
UI_CCL_MANIFESTS := pkg/ui/protos.ccl.manifest.json pkg/ui/vendor.oss.manifest.json
UI_CCL_DLLS := pkg/ui/workspaces/db-console/dist/protos.ccl.dll.js pkg/ui/workspaces/db-console/dist/vendor.oss.dll.js
UI_CCL_MANIFESTS := pkg/ui/workspaces/db-console/protos.ccl.manifest.json pkg/ui/workspaces/db-console/vendor.oss.manifest.json
UI_OSS_DLLS := $(subst .ccl,.oss,$(UI_CCL_DLLS))
UI_OSS_MANIFESTS := $(subst .ccl,.oss,$(UI_CCL_MANIFESTS))

Expand All @@ -1383,41 +1360,41 @@ UI_OSS_MANIFESTS := $(subst .ccl,.oss,$(UI_CCL_MANIFESTS))
# [1]: http://savannah.gnu.org/bugs/?19108
.SECONDARY: $(UI_CCL_DLLS) $(UI_CCL_MANIFESTS) $(UI_OSS_DLLS) $(UI_OSS_MANIFESTS)

pkg/ui/dist/%.oss.dll.js pkg/ui/%.oss.manifest.json: pkg/ui/webpack.%.js pkg/ui/yarn.installed $(UI_PROTOS_OSS) $(CLUSTER_UI_JS)
$(NODE_RUN) -C pkg/ui $(WEBPACK) -p --config webpack.$*.js --env.dist=oss
pkg/ui/workspaces/db-console/dist/%.oss.dll.js pkg/ui/workspaces/db-console/%.oss.manifest.json: pkg/ui/workspaces/db-console/webpack.%.js pkg/ui/yarn.installed $(CLUSTER_UI_JS) $(UI_PROTOS_OSS)
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(WEBPACK) -p --config webpack.$*.js --env.dist=oss

pkg/ui/dist/%.ccl.dll.js pkg/ui/%.ccl.manifest.json: pkg/ui/webpack.%.js pkg/ui/yarn.installed $(UI_PROTOS_CCL) $(CLUSTER_UI_JS)
$(NODE_RUN) -C pkg/ui $(WEBPACK) -p --config webpack.$*.js --env.dist=ccl
pkg/ui/workspaces/db-console/dist/%.ccl.dll.js pkg/ui/workspaces/db-console/%.ccl.manifest.json: pkg/ui/workspaces/db-console/webpack.%.js pkg/ui/yarn.installed $(CLUSTER_UI_JS) $(UI_PROTOS_CCL)
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(WEBPACK) -p --config webpack.$*.js --env.dist=ccl

.PHONY: ui-test
ui-test: $(UI_CCL_DLLS) $(UI_CCL_MANIFESTS)
$(NODE_RUN) -C pkg/ui $(KARMA) start
$(NODE_RUN) -C pkg/ui/cluster-ui yarn ci
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(KARMA) start
$(NODE_RUN) -C pkg/ui/workspaces/cluster-ui yarn ci

.PHONY: ui-test-watch
ui-test-watch: $(UI_CCL_DLLS) $(UI_CCL_MANIFESTS)
$(NODE_RUN) -C pkg/ui $(KARMA) start --no-single-run --auto-watch & \
$(NODE_RUN) -C pkg/ui/cluster-ui yarn test
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(KARMA) start --no-single-run --auto-watch & \
$(NODE_RUN) -C pkg/ui/workspaces/cluster-ui yarn test

.PHONY: ui-test-debug
ui-test-debug: $(UI_DLLS) $(UI_MANIFESTS)
$(NODE_RUN) -C pkg/ui $(KARMA) start --browsers Chrome --no-single-run --debug --auto-watch
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(KARMA) start --browsers Chrome --no-single-run --debug --auto-watch

pkg/ui/distccl/bindata.go: $(UI_CCL_DLLS) $(UI_CCL_MANIFESTS) $(UI_JS_CCL) $(shell find pkg/ui/ccl -type f)
pkg/ui/distccl/bindata.go: $(UI_CCL_DLLS) $(UI_CCL_MANIFESTS) $(UI_JS_CCL) $(shell find pkg/ui/workspaces/db-console/ccl -type f)
pkg/ui/distoss/bindata.go: $(UI_OSS_DLLS) $(UI_OSS_MANIFESTS) $(UI_JS_OSS)
pkg/ui/dist%/bindata.go: pkg/ui/webpack.app.js $(shell find pkg/ui/src pkg/ui/styl -type f) | bin/.bootstrap
pkg/ui/dist%/bindata.go: pkg/ui/workspaces/db-console/webpack.app.js $(CLUSTER_UI_JS) $(shell find pkg/ui/workspaces/db-console/src pkg/ui/workspaces/db-console/styl -type f) | bin/.bootstrap
find pkg/ui/dist$* -mindepth 1 -not -name dist$*.go -delete
set -e; shopt -s extglob; for dll in $(notdir $(filter %.dll.js,$^)); do \
ln -s ../dist/$$dll pkg/ui/dist$*/$${dll/@(.ccl|.oss)}; \
ln -s ../workspaces/db-console/dist/$$dll pkg/ui/dist$*/$${dll/@(.ccl|.oss)}; \
done
$(NODE_RUN) -C pkg/ui $(WEBPACK) --config webpack.app.js --env.dist=$*
$(NODE_RUN) -C pkg/ui/workspaces/db-console $(WEBPACK) --config webpack.app.js --env.dist=$*
go-bindata -pkg dist$* -o $@ -prefix pkg/ui/dist$* pkg/ui/dist$*/...
echo 'func init() { ui.Asset = Asset; ui.AssetDir = AssetDir; ui.AssetInfo = AssetInfo }' >> $@
gofmt -s -w $@
goimports -w $@

pkg/ui/yarn.opt.installed:
$(NODE_RUN) -C pkg/ui/opt yarn install
$(NODE_RUN) -C pkg/ui yarn install --check-files
touch $@

.PHONY: ui-watch-secure
Expand All @@ -1433,20 +1410,20 @@ ui-watch ui-watch-secure: $(UI_CCL_DLLS) pkg/ui/yarn.opt.installed
#
# `node-run.sh` wrapper is removed because this command is supposed to be run in dev environment (not in docker of CI)
# so it is safe to run yarn commands directly to preserve formatting and colors for outputs
yarn --cwd pkg/ui/cluster-ui build:watch & \
yarn --cwd pkg/ui webpack-dev-server --config webpack.app.js --env.dist=ccl --port $(PORT) --mode "development" $(WEBPACK_DEV_SERVER_FLAGS)
yarn --cwd pkg/ui/workspaces/cluster-ui build:watch & \
yarn --cwd pkg/ui/workspaces/db-console webpack-dev-server --config webpack.app.js --env.dist=ccl --port $(PORT) --mode "development" $(WEBPACK_DEV_SERVER_FLAGS)

.PHONY: ui-clean
ui-clean: ## Remove build artifacts.
find pkg/ui/dist* -mindepth 1 -not -name dist*.go -delete
find pkg/ui/workspaces/db-console/dist* -mindepth 1 -not -name dist*.go -delete
rm -f $(UI_PROTOS_CCL) $(UI_PROTOS_OSS)
rm -f pkg/ui/*manifest.json
rm -rf pkg/ui/cluster-ui/dist
rm -f pkg/ui/workspaces/db-console/*manifest.json
rm -rf pkg/ui/workspaces/cluster-ui/dist

.PHONY: ui-maintainer-clean
ui-maintainer-clean: ## Like clean, but also remove installed dependencies
ui-maintainer-clean: ui-clean
rm -rf pkg/ui/node_modules pkg/ui/yarn.installed pkg/ui/cluster-ui/node_modules pkg/ui/yarn.cluster-ui.installed pkg/ui/src/js/node_modules pkg/ui/yarn.protobuf.installed
rm -rf pkg/ui/node_modules pkg/ui/workspaces/db-console/node_modules pkg/ui/yarn.installed pkg/ui/workspaces/cluster-ui/node_modules

.SECONDARY: pkg/sql/parser/gen/sql.go.tmp
pkg/sql/parser/gen/sql.go.tmp: pkg/sql/parser/gen/sql-gen.y bin/.bootstrap
Expand Down Expand Up @@ -1872,4 +1849,4 @@ $(foreach v,$(filter-out $(strip $(VALID_VARS)),$(.VARIABLES)),\
# Cypress e2e tests
.PHONY: db-console-e2e-test
db-console-e2e-test: pkg/ui/yarn.opt.installed
cd pkg/ui && yarn cypress:run
cd pkg/ui/workspaces/db-console && yarn cypress:run
4 changes: 0 additions & 4 deletions pkg/ui/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,3 @@ dist*/**
!distccl/distccl.go
!distoss/distoss.go
*manifest.json

cypress/screenshots/*
cypress/videos/*
cypress/snapshots/**/__diff_output__
1 change: 0 additions & 1 deletion pkg/ui/.yarnrc
View file

This file was deleted.

Loading

0 comments on commit 1b47ae3

Please sign in to comment.