Merge branch 'conc1-when-ratelim-set' of https://github.com/yujuncen/br…

… into conc1-when-ratelim-set

.github/workflows/compatible_test.yml

@@ -0,0 +1,66 @@
+name: compatibility-test
+
+on:
+  push:
+    branches:
+      - master
+      - 'release-[0-9].[0-9]*'
+  pull_request:
+    branches:
+      - master
+      - 'release-[0-9].[0-9]*'
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+
+    - uses: khan/pull-request-comment-trigger@master
+      id: check
+      with:
+        trigger: '/run-compatiblility-tests'
+
+    - name: Free disk space
+      run: |
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /usr/share/dotnet
+        docker volume prune -f
+        docker image prune -f
+      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+
+    - uses: actions/checkout@v2
+      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.15
+      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+
+    - name: Generate compatibility test backup data
+      timeout-minutes: 15
+      run: sh compatibility/prepare_backup.sh
+      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+
+    - name: Start server
+      run: |
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml rm -s -v
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml build
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml up --remove-orphans -d
+        TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml exec -T control make compatibility_test
+      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+
+    - name: Collect component log
+      if: ${{ failure() }}
+      run: |
+        tar czvf /tmp/br/logs.tar.gz /tmp/br/docker/backup_logs/*
+
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: logs
+        path: /tmp/br/logs.tar.gz

COMPATIBILITY_TEST.md

@@ -0,0 +1,42 @@
+# Compatibility test
+
+## Background
+
+We had some incompatibility issues in the past, which made BR cannot restore backed up data in some situations.
+So we need a test workflow to check the compatiblity.
+
+## Goal
+
+- Ensure backward compatibility for restoring data from the previous 3 minor versions
+
+## Workflow
+
+### Data Preparation
+
+This workflow needs previous backup data. To get this data. we perform the following steps
+
+- Run a TiDB cluster with previous version.
+- Run backup jobs with corresponding BR version, with different storages (s3, gcs).
+
+Given we test for the previous 3 versions, and there are 2 different storage systems, we will produce 6 backup archives for 6 separate compatibility tests.
+
+### Test Content
+
+- Start TiDB cluster with nightly version.
+- Build BR binary with current directory.
+- Use BR to restore different version backup data one by one.
+- Make sure restore data is expected.
+
+### Running tests
+
+Start a cluster with docker-compose and Build br with latest version.
+
+```sh
+docker-compose -f docker-compose.yaml rm -s -v && \
+docker-compose -f docker-compose.yaml build && \
+docker-compose -f docker-compose.yaml up --remove-orphans
+```
+
+```sh
+docker-compose -f docker-compose.yaml control make compatibility_test
+```

Makefile

@@ -11,7 +11,7 @@ BR_PKG := github.com/pingcap/br
 
 VERSION := v5.0.0-master
 release_version_regex := ^v5\..*$$
-release_branch_regex := "^release-[0-9]\.[0-9].*$$|^HEAD$$"
+release_branch_regex := "^release-[0-9]\.[0-9].*$$|^HEAD$$|*/tags/v[0-9]\.[0-9]\."
 ifneq ($(shell git rev-parse --abbrev-ref HEAD | egrep $(release_branch_regex)),)
 	# If we are in release branch, try to use tag version.
 	ifneq ($(shell git describe --tags --dirty | egrep $(release_version_regex)),)
@@ -126,6 +126,12 @@ testcover: tools
 integration_test: bins build build_for_integration_test
 	tests/run.sh
 
+compatibility_test_prepare:
+	tests/run_compatible.sh prepare
+
+compatibility_test: br
+	tests/run_compatible.sh run
+
 coverage: tools
 	tools/bin/gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
 ifeq ("$(JenkinsCI)", "1")
@@ -213,12 +219,12 @@ static: prepare tools
 		$(PACKAGE_DIRECTORIES)
 	# pingcap/errors APIs are mixed with multiple patterns 'pkg/errors',
 	# 'juju/errors' and 'pingcap/parser'. To avoid confusion and mistake,
-	# we only allow a subset of APIs, that's "Normalize|Annotate|Trace|Cause".
+	# we only allow a subset of APIs, that's "Normalize|Annotate|Trace|Cause|Find".
 	# TODO: check lightning packages.
 	@# TODO: allow more APIs when we need to support "workaound".
 	grep -Rn --include="*.go" --exclude="*_test.go" -E "(\t| )errors\.[A-Z]" \
 		$$($(PACKAGE_DIRECTORIES) | grep -vE "tests|lightning") | \
-		grep -vE "Normalize|Annotate|Trace|Cause|RedactLogEnabled" 2>&1 | $(CHECKER)
+		grep -vE "Normalize|Annotate|Trace|Cause|RedactLogEnabled|Find" 2>&1 | $(CHECKER)
 	# The package name of "github.com/pingcap/kvproto/pkg/backup" collides
 	# "github.com/pingcap/br/pkg/backup", so we rename kvproto to backuppb.
 	grep -Rn --include="*.go" -E '"github.com/pingcap/kvproto/pkg/backup"' \

README.md

@@ -87,6 +87,10 @@ bin/br restore full --pd pd0:2379 --storage "s3://mybucket/full" \
     --s3.endpoint="$S3_ENDPOINT"
 ```
 
+## Compatible test
+
+See [COMPATBILE_TEST](./COMPATIBLE_TEST.md)
+
 ## Contributing
 
 Contributions are welcomed and greatly appreciated. See [CONTRIBUTING](./CONTRIBUTING.md)

cmd/br/debug.go

@@ -269,7 +269,7 @@ func decodeBackupMetaCommand() *cobra.Command {
 			fieldName, _ := cmd.Flags().GetString("field")
 			if fieldName == "" {
 				// No field flag, write backupmeta to external storage in JSON format.
-				backupMetaJSON, err := json.Marshal(backupMeta)
+				backupMetaJSON, err := utils.MarshalBackupMeta(backupMeta)
 				if err != nil {
 					return errors.Trace(err)
 				}
@@ -332,8 +332,7 @@ func encodeBackupMetaCommand() *cobra.Command {
 				return errors.Trace(err)
 			}
 
-			backupMetaJSON := &backuppb.BackupMeta{}
-			err = json.Unmarshal(metaData, backupMetaJSON)
+			backupMetaJSON, err := utils.UnmarshalBackupMeta(metaData)
 			if err != nil {
 				return errors.Trace(err)
 			}

compatibility/backup_cluster.yaml

@@ -0,0 +1,128 @@
+---
+# Source: tidb-docker-compose/templates/docker-compose.yml
+version: '3.2'
+
+services:
+  control:
+    image: control:nightly
+    build:
+      context: ../
+      dockerfile: ./docker/Dockerfile
+    volumes:
+      - /tmp/br/docker/backup_data/${TAG}:/data
+      - /tmp/br/docker/backup_logs/${TAG}:/tmp
+      - /tmp/br/docker/backup_logs/${TAG}:/logs
+      - ../bin:/go/src/github.com/pingcap/br/bin
+      - ./prepare_data:/prepare_data
+    command: -c "/usr/bin/tail -f /dev/null"
+    environment:
+      TAG: ${TAG}
+    depends_on:
+      - "tidb"
+    restart: on-failure
+    env_file:
+      - ../docker/minio.env
+
+  pd0:
+    image: pingcap/pd:${TAG}
+    ports:
+      - 237${PORT_SUFFIX}:2379
+    volumes:
+      - ../docker/config/pd.toml:/pd.toml:ro
+      - /tmp/br/docker/backup_data/${TAG}:/data
+      - /tmp/br/docker/backup_logs/${TAG}:/logs
+    command:
+      - --name=pd0
+      - --client-urls=http://0.0.0.0:2379
+      - --peer-urls=http://0.0.0.0:2380
+      - --advertise-client-urls=http://pd0:2379
+      - --advertise-peer-urls=http://pd0:2380
+      - --initial-cluster=pd0=http://pd0:2380
+      - --data-dir=/data/pd0
+      - --config=/pd.toml
+      - --log-file=/logs/pd0.log
+    # sysctls:
+    #   net.core.somaxconn: 32768
+    # ulimits:
+    #   nofile:
+    #     soft: 1000000
+    #     hard: 1000000
+    restart: on-failure
+
+  tikv0:
+    image: pingcap/tikv:${TAG}
+    volumes:
+      - ../docker/config/tikv.toml:/tikv.toml:ro
+      - /tmp/br/docker/backup_data/${TAG}:/data
+      - /tmp/br/docker/backup_logs/${TAG}:/logs
+    command:
+      - --addr=0.0.0.0:20160
+      - --advertise-addr=tikv0:20160
+      - --data-dir=/data/tikv0
+      - --pd=pd0:2379
+      - --config=/tikv.toml
+      - --log-file=/logs/tikv0.log
+    depends_on:
+      - "pd0"
+    # sysctls:
+    #   net.core.somaxconn: 32768
+    # ulimits:
+    #   nofile:
+    #     soft: 1000000
+    #     hard: 1000000
+    restart: on-failure
+    env_file:
+      - ../docker/minio.env
+
+  tidb:
+    image: pingcap/tidb:${TAG}
+    ports:
+      - 400${PORT_SUFFIX}:4000
+      - 1008${PORT_SUFFIX}:10080
+    volumes:
+      - ../docker/config/tidb.toml:/tidb.toml:ro
+      - /tmp/br/docker/backup_logs/${TAG}:/logs
+      - ./credentials:/credentials
+    command:
+      - --store=tikv
+      - --path=pd0:2379
+      - --config=/tidb.toml
+      - --log-file=/logs/tidb.log
+      - --advertise-address=tidb
+    depends_on:
+      - "tikv0"
+    restart: on-failure
+    env_file:
+      - ../docker/gcs.env
+
+  minio:
+    image: minio/minio
+    ports:
+      - 2492${PORT_SUFFIX}:24927
+    volumes:
+      - /tmp/br/docker/data/s3:/data/s3
+    command:
+      - server
+      - --address=:24927
+      - /data/s3
+    env_file:
+      - ../docker/minio.env
+
+  gcs:
+    image: fsouza/fake-gcs-server
+    ports:
+      - 2081${PORT_SUFFIX}:20818
+    volumes:
+      - /tmp/br/docker/backup_data/gcs:/data/gcs
+    command:
+      - -scheme=http
+      - -host=gcs
+      - -port=20818
+      - -filesystem-root=/data/gcs
+      - -public-host=gcs:20818
+
+  # we need a oauth server for gcs test
+  oauth:
+    image: 3pointer/fake-gcs-oauth
+    ports:
+      - 500${PORT_SUFFIX}:5000

compatibility/credentials/application_default_credentials.json

@@ -0,0 +1,6 @@
+{
+  "type": "service_account",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCT524vzG7uEVtX\nojcHbyQzVwlcaGkg1DWWLT+SufD08UYF0bsfcD0Etrtzo4ggwdxJQy5ygl3TNlcD\nKdelWbVyGfg9/sNB1RDlZYbQb0LVLHKjkVs7JyJsxrLk2e6NqD9ajwTEJUcLAQkj\nxlCcIi51beqrIRlvHjbtGwet/dNnRLSZf+i9SHvB2j64+RVYdnyf/IiLBvYyu7hF\nT6VjlljdbwC4TZ2jpfDL8nHRTiDiV+CX3/iH8MlMEOSM30AO5MPNVCZLlTA9W24a\nKi4NPBBlJLvG2mQELYdbhdM64iMvbPkDRtajJD6ogPB7wUoWbtSke5oOJNyV1HNt\nn91JH/dlAgMBAAECggEAQBwve2GSbfgxD0Xds4e9+dEO2jLZ6uSBS9TWOywFIa9Z\nqlkUUtbMZDgu/buTXJubeFg6EGGo+M4TnmfrNR2zFD/khj7hdS49kinVa5Dmt895\n66Osl3HprpvcXG2IxXd56q+Woc0Ew+TRiOPD+kGowLcB4ubIhw1iQpmWVRlyos6Q\nyvHssolrqOkRK9+1asixgow2Y15HtpXFN3XDIVj3gfdN1Zg80S66bTap1DS+dkJH\nSMgEZRilAjUGzbroqvZCiymlIJP5Jj5L5Wy8Qp/k1ixK10oaPgwvdmwXHX/DZ0vC\nT6XwpIaCYd3/XUWBHvrmQHFucWVPISZRi5WidggzuwKBgQDNHrxKaDrxcrV5Ncgu\npQrtQvTsIUCJGMo5m30X0Ac5CsIssOoQHdtEQW1ehJ8DtJRRb9rdWc4aelXsDUr+\no2m1zyZzM6S7IO2YhGDAo7Uu3fy1r33qYAt6uS/nHaJBpsKcyqqK+0wPDikdPLLx\nBBWZHF6WoswDEUVLQa/hHgpjPwKBgQC4l2/6xShNoobivzk8AE/Acq7PazA8gu4K\nY0UghTBlAst4RvBTURYZ2V3uw0S2FbfwL0/snHhNWZl5XjBX/H9oQmLri5qGOOpf\n9A11p5kd0x1mHDgTm/k7EgoskdXGB5NqXIB7l/3UI8Sk2N1PzHwyJJYfaB+EWTs8\n+LVy99VQWwKBgQCilRwVtiwSOSPSYWi8YCEbEpljmK+4eye/JZmviDpRYk+qcMf1\n4lRr85gm9OO9YiK1sf0+ufH9Vr5IDflFgG1HqFwHsAWANYdd/n9Z8eior1ehAurB\nHUO8EJEBlaGIfA+Bi7pF0w3kWQsJm5USKHSeGbh3ma4vOD8+eWBZBSCirQKBgQCe\n1uEq/sChnXtIXpgXg4Uc6xJ1tZy6VUgUdDulsjZklTUU+KYQa7QC5kKoFCtqK+It\nseiqiDIVDUa9Y0liTQotYwLQAT8kxJEZpF54oZFmUqX3mcy/QvYB2JIcrBkx4I7/\ndT2yHKX1CBpMZ7h41FMCquzrdaO5NTd+Td2FYrGSBQKBgEBnAerHh/NafYlVumlS\nVgouR9IketTegyEyntVyEvENx8OA5ZLMywCIKbPMFZgPR0RgDpyDxKauCU2E09e/\nboN76UOuOg11fknJh7vFbUbzM6BXvXVOTyX9ZtZBQcd5Y3tV+tYD1tHUgurGYWb+\nyHLBMOlXdpn0gZ4rwoIQgzD9\n-----END PRIVATE KEY-----\n",
+  "client_email": "test@email.com",
+  "token_uri": "http://oauth:5000/oauth/token"
+}

compatibility/prepare_backup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+# update tags
+git fetch --tags
+
+TAGS="v5.0.0"
+getLatestTags() {
+  release_5_branch_regex="^release-5\.[0-9].*$"
+  release_4_branch_regex="^release-4\.[0-9].*$"
+  TOTAL_TAGS=$(git for-each-ref --sort=creatordate  refs/tags | awk -F '/' '{print $3}')
+  # latest tags
+  TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | tail -n3)
+  if git rev-parse --abbrev-ref HEAD | egrep -q $release_5_branch_regex
+  then
+    # If we are in release-5.0 branch, try to use latest 3 version of 5.x and last 4.x version
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n1 && echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v5." | tail -n3)
+  elif git rev-parse --abbrev-ref HEAD | egrep -q $release_4_branch_regex
+  then
+    # If we are in release-4.0 branch, try to use latest 3 version of 4.x
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n3)
+  fi
+}
+
+getLatestTags
+echo "recent version of cluster is $TAGS"
+
+i=0
+
+runBackup() {
+  # generate backup data in /tmp/br/docker/backup_data/$TAG/, we can do restore after all data backuped later.
+  echo "build $1 cluster"
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml build
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml up -d
+  trap "TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml down" EXIT
+  # wait for cluster ready
+  sleep 20
+  # prepare SQL data
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml exec -T control /go/bin/go-ycsb load mysql -P /prepare_data/workload -p mysql.host=tidb -p mysql.port=4000 -p mysql.user=root -p mysql.db=test
+  TAG=$1 PORT_SUFFIX=$2 docker-compose -p $1 -f compatibility/backup_cluster.yaml exec -T control make compatibility_test_prepare
+}
+
+for tag in $TAGS; do
+   i=$(( i + 1 ))
+   runBackup $tag $i &
+done
+
+wait
+
+echo "prepare backup data successfully"

compatibility/prepare_data/workload

@@ -0,0 +1,11 @@
+recordcount=1000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+