Merge branch 'master' into simulate-state-change

.circleci/config.yml

@@ -47,19 +47,20 @@ executors:
     resource_class: arm.large
   mac_amd64_medium:
     macos:
-      xcode: 13.2.1
-    resource_class: medium
+      xcode: 14.2.0
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_amd64_large:
     macos:
-      xcode: 13.2.1
-    resource_class: large
+      xcode: 14.2.0
+    # Since they removed the large class for amd64, we will use medium here too.
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_arm64: &executor-mac-arm64
     machine: true
-    resource_class: algorand/macstadium-m1-macos11
+    resource_class: algorand/macstadium-m1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   # these are required b/c jobs explicitly assign sizes to the executors
@@ -456,7 +457,7 @@ commands:
           shell: bash.exe
           command: |
             choco install -y msys2 pacman make wget --force
-            choco install -y golang --version=1.20.5 --force
+            choco install -y golang --version=1.20.6 --force
             choco install -y python3 --version=3.7.3 --force
             export msys2='cmd //C RefreshEnv.cmd '
             export msys2+='& set MSYS=winsymlinks:nativestrict '
@@ -601,7 +602,6 @@ commands:
             export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
             export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
             export PARTITION_ID=${CIRCLE_NODE_INDEX}
-            export GOEXPERIMENT="none"
             gotestsum --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
       - store_artifacts:
           path: << parameters.result_path >>

.github/workflows/pr-type-category.yml

@@ -10,7 +10,7 @@ jobs:
     name: Check PR Category and Type
     steps:
       - name: Checking for correct number of required github pr labels
-        uses: mheap/github-action-required-labels@v2
+        uses: mheap/github-action-required-labels@v5
         with:
           mode: exactly
           count: 1

.github/workflows/reviewdog.yml

@@ -56,7 +56,7 @@ jobs:
         uses: actions/cache@v3.3.1
         with:
           path: cicdtmp/golangci-lint/golangci-lint-cgo
-          key: cicd-golangci-lint-cgo-v0.0.2
+          key: cicd-golangci-lint-cgo-v0.0.2-${{ env.GO_VERSION }}
 
       - name: Build custom golangci-lint with CGO_ENABLED
         if: steps.cache-golangci-lint.outputs.cache-hit != 'true'

.github/workflows/tools.yml

@@ -9,9 +9,6 @@ on:
       - 'tools/block-generator/**'
       - 'tools/x-repo-types/**'
   pull_request:
-    paths:
-      - 'tools/block-generator/**'
-      - 'tools/x-repo-types/**'
 
 jobs:
   tools_test:

Dockerfile

@@ -1,6 +1,6 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:20.04 as builder
 
-ARG GO_VERSION="1.20.5"
+ARG GO_VERSION="1.20.6"
 
 ARG CHANNEL
 ARG URL

Makefile

@@ -49,24 +49,16 @@ export CPATH=/opt/homebrew/include
 export LIBRARY_PATH=/opt/homebrew/lib
 endif
 endif
+
 ifeq ($(UNAME), Linux)
 EXTLDFLAGS := -static-libstdc++ -static-libgcc
-ifeq ($(ARCH), amd64)
 # the following predicate is abit misleading; it tests if we're not in centos.
 ifeq (,$(wildcard /etc/centos-release))
 EXTLDFLAGS  += -static
 endif
 GOTAGSLIST  += osusergo netgo static_build
 GOBUILDMODE := -buildmode pie
 endif
-ifeq ($(ARCH), arm)
-ifneq ("$(wildcard /etc/alpine-release)","")
-EXTLDFLAGS  += -static
-GOTAGSLIST  += osusergo netgo static_build
-GOBUILDMODE := -buildmode pie
-endif
-endif
-endif
 
 ifneq (, $(findstring MINGW,$(UNAME)))
 EXTLDFLAGS := -static -static-libstdc++ -static-libgcc
@@ -77,11 +69,6 @@ ifeq ($(SHORT_PART_PERIOD), 1)
 export SHORT_PART_PERIOD_FLAG := -s
 endif
 
-# Disable go experiments during build as of go 1.20.5 due to
-# https://github.com/golang/go/issues/60825
-# Likely fix: https://go-review.googlesource.com/c/go/+/503937/6/src/runtime/race_arm64.s
-export GOEXPERIMENT=none
-
 GOTAGS      := --tags "$(GOTAGSLIST)"
 GOTRIMPATH	:= $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath)
 
@@ -99,7 +86,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d
 
 GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 
@@ -220,9 +207,11 @@ build: buildsrc buildsrc-special
 # get around a bug in go build where it will fail
 # to cache binaries from time to time on empty NFS
 # dirs
-buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN
-	mkdir -p "${GOCACHE}" && \
-	touch "${GOCACHE}"/file.txt && \
+${GOCACHE}/file.txt:
+	mkdir -p "${GOCACHE}"
+	touch "${GOCACHE}"/file.txt
+
+buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt
 	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 buildsrc-special:

README.md

@@ -22,7 +22,7 @@ the [official Go documentation website](https://golang.org/doc/).
 
 ### Linux / OSX
 
-We currently strive to support Debian-based distributions with Ubuntu 22.04
+We currently strive to support Debian-based distributions with Ubuntu 20.04
 being our official release target.
 Building on Arch Linux works as well.
 Our core engineering team uses Linux and OSX, so both environments are well

agreement/fuzzer/networkFacade_test.go

@@ -77,7 +77,7 @@ func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade {
 	n := &NetworkFacade{
 		fuzzer:         fuzzer,
 		nodeID:         nodeID,
-		mux:            network.MakeMultiplexer(fuzzer.log),
+		mux:            network.MakeMultiplexer(),
 		clocks:         make(map[int]chan time.Time),
 		eventsQueues:   make(map[string]int),
 		eventsQueuesCh: make(chan int, 1000),

agreement/gossip/network_test.go

@@ -323,7 +323,7 @@ func makewhiteholeNetwork(domain *whiteholeDomain) *whiteholeNetwork {
 	w := &whiteholeNetwork{
 		peer:         atomic.AddUint32(&domain.peerIdx, 1),
 		lastMsgRead:  uint32(len(domain.messages)),
-		mux:          network.MakeMultiplexer(domain.log),
+		mux:          network.MakeMultiplexer(),
 		domain:       domain,
 		disconnected: make(map[uint32]bool),
 	}

agreement/message.go

@@ -28,7 +28,7 @@ type message struct {
 
 	// this field is for backwards compatibility with crash state serialized using go-codec prior to explicit unexport.
 	// should be removed after the next consensus update.
-	MessageHandle msgp.Raw
+	MessageHandle msgp.Raw `codec:"MessageHandle,omitempty"`
 	// explicitly unexport this field since we can't define serializers for interface{} type
 	// the only implementation of this is gossip.messageMetadata which doesn't have exported fields to serialize.
 	messageHandle MessageHandle

agreement/message_test.go

@@ -111,6 +111,7 @@ func TestMessageBackwardCompatibility(t *testing.T) {
 		Tag: protocol.ProposalPayloadTag,
 	}
 
+	require.Containsf(t, string(encoded), "MessageHandle", "encoded message does not contain MessageHandle field")
 	var m1, m2, m3, m4 message
 	// Both msgp and reflection should decode the message containing old MessageHandle successfully
 	err = protocol.Decode(encoded, &m1)
@@ -123,6 +124,7 @@ func TestMessageBackwardCompatibility(t *testing.T) {
 	e1 := protocol.Encode(&m1)
 	e2 := protocol.EncodeReflect(&m2)
 	require.Equal(t, e1, e2)
+	require.NotContainsf(t, string(e1), "MessageHandle", "encoded message still contains MessageHandle field")
 	err = protocol.DecodeReflect(e1, &m3)
 	require.NoError(t, err)
 	err = protocol.Decode(e2, &m4)

catchup/catchpointService_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/test/partitiontest"
 )
@@ -76,6 +77,11 @@ func (m *catchpointCatchupAccessorMock) Ledger() (l ledger.CatchupAccessorClient
 	return m.l
 }
 
+// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
+func (m *catchpointCatchupAccessorMock) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
+	return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, nil
+}
+
 // TestCatchpointServicePeerRank ensures CatchpointService does not crash when a block fetched
 // from the local ledger and not from network when ranking a peer
 func TestCatchpointServicePeerRank(t *testing.T) {

catchup/service.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -744,17 +745,19 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy
 		if cert.Round == fetchedCert.Round &&
 			cert.Proposal.BlockDigest != fetchedCert.Proposal.BlockDigest &&
 			fetchedCert.Authenticate(*block, s.ledger, verifier) == nil {
-			s := "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "fetchRound called with a cert authenticating block with hash %v.\n"
-			s += "We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n"
-			s += "Cert from our agreement service:\n%#v\n\n"
-			s += "Cert from the fetcher:\n%#v\n\n"
-			s += "Block from the fetcher:\n%#v\n\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+			var builder strings.Builder
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("fetchRound called with a cert authenticating block with hash %v.\n")
+			builder.WriteString("We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n")
+			builder.WriteString("Cert from our agreement service:\n%#v\n\n")
+			builder.WriteString("Cert from the fetcher:\n%#v\n\n")
+			builder.WriteString("Block from the fetcher:\n%#v\n\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			s := builder.String()
 			s = fmt.Sprintf(s, cert.Proposal.BlockDigest, fetchedCert.Proposal.BlockDigest, cert, fetchedCert, block)
 			fmt.Println(s)
 			logging.Base().Error(s)

cmd/algons/commands.go

@@ -25,6 +25,7 @@ import (
 
 func init() {
 	rootCmd.AddCommand(dnsCmd)
+	rootCmd.AddCommand(dnsaddrCmd)
 }
 
 var rootCmd = &cobra.Command{