test sharding

.github/workflows/run-acceptance-tests.yml

@@ -179,7 +179,7 @@ jobs:
           status: ${{ job.status }}
 
   test:
-    runs-on: pulumi-ubuntu-8core
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       id-token: write
@@ -189,12 +189,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        language:
-          - nodejs
-          - python
-          - dotnet
-          - go
-          - java
+        shard: [0, 1, 2, 3, 4, 5, 6, 7]
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -233,7 +228,7 @@ jobs:
         run: yarn global add typescript
       - run: dotnet nuget add source ${{ github.workspace }}/nuget
       - name: Install dependencies
-        run: make install_${{ matrix.language}}_sdk
+        run: mise run install
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -254,12 +249,11 @@ jobs:
       - name: Setup gcloud auth
         uses: google-github-actions/setup-gcloud@v2
         with:
-          install_components: gke-gcloud-auth-plugin
+          skip_install: true
+      - name: Shard tests
+        run: mise run "test:examples:shard" -- -n ${{ strategy.job-total }} --shard ${{ strategy.job-index }} > go-test
       - name: Run tests
-        run: >-
-          set -euo pipefail
-
-          cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: mise run "test:examples" $(cat go-test) 2>&1 | tee /tmp/gotest.log | gotestfmt
       - if: failure() && github.event_name == 'push'
         name: Notify Slack
         uses: 8398a7/action-slack@v3

.mise.toml

@@ -1,5 +1,5 @@
 [tasks.provider]
-run = "make provider"
+run = ["make provider"]
 sources = ["provider/**/*.go", "go.mod"]
 outputs = ["bin/pulumi-resource-docker-build"]
 
@@ -67,9 +67,42 @@ run = "make build_java"
 sources = ["bin/pulumi-resource-docker-build"]
 outputs = ["*"]
 
+# TODO: Might be able to link these directly?
+[tasks."install:dotnet"]
+run = "make install_dotnet_sdk"
+
+[tasks."install:nodejs"]
+run = "make install_nodejs_sdk"
+
 [tasks.sdk]
 depends = ["generate:*", "build:*"]
 
+[tasks.install]
+depends = ["install:*"]
+
+[tasks.test]
+sources = ["**/*.go"]
+outputs = ["*"]
+run = "make test_provider"
+
+[tasks."test:examples"]
+run = "go test -v -json -cover -timeout 2h -tags=all -parallel 4"
+
+[tasks."test:examples:shard"]
+run = "go run ./enumerate/main.go --root ./examples"
+
+[tasks.schema]
+run = "pulumi package get-schema bin/pulumi-resource-docker-build | jq 'del(.version)' > provider/cmd/pulumi-resource-docker-build/schema.json"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["provider/cmd/pulumi-resource-docker-build/schema.json"]
+
+[tasks.docs]
+run = "go generate docs/generate.go"
+depends = ["schema"]
+sources = ["docs/yaml/*.yaml"]
+outputs= ["provider/internal/embed/*.md"]
+
+# TODO: tasks.examples
 
 [tools]
 # Development tooling.
@@ -87,6 +120,8 @@ gradle = "7.6"
 "go:github.com/pulumi/schema-tools" = "v0.6.0"
 "go:github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt" = "v2.5.0"
 "go:github.com/pulumi/pulumictl/cmd/pulumictl" = "a1b89aaf4299fe5bc78e140fc60eba79088b3dc5" # blampe/replace branch doesn't work?
+yarn = "1.22.19"
+gcloud = "455.0.0"
 
 [env]
 # TODO(https://github.com/jdx/mise/issues/1898): create = true

enumerate/main.go

@@ -0,0 +1,121 @@
+// Copyright 2024, Pulumi Corporation.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+)
+
+var (
+	rootFlag  = flag.String("root", ".", "directory to search")
+	shardFlag = flag.Int("shard", 0, "shard index to collect tests for")
+	nFlag     = flag.Int("n", 1, "the total number of shards")
+	seedFlag  = flag.Int64("seed", 0, "randomly shuffle tests using a non-zero seed")
+)
+
+var re = regexp.MustCompile(`^Test[A-Z_]`)
+
+type testf struct {
+	path string
+	name string
+}
+
+func main() {
+	flag.Parse()
+
+	root := *rootFlag
+	shard := *shardFlag
+	n := *nFlag
+	if shard >= n {
+		log.Fatal("shard must be less than n")
+	}
+	if shard < 0 || n < 0 {
+		log.Fatal("must be non-negative")
+	}
+
+	tests := []testf{}
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() || !strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+
+		// Parse the file to find test functions
+		fileSet := token.NewFileSet()
+		node, err := parser.ParseFile(fileSet, path, nil, 0)
+		if err != nil {
+			return err
+		}
+		for _, decl := range node.Decls {
+			f, ok := decl.(*ast.FuncDecl)
+			if !ok {
+				continue
+			}
+			name := f.Name.Name
+			if !re.MatchString(name) {
+				continue
+			}
+			tests = append(tests, testf{path: filepath.Dir(path), name: name})
+		}
+
+		return nil
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Shuffle the tests.
+	seed := *seedFlag
+	if seed != 0 {
+		random := rand.New(rand.NewSource(seed))
+		for i := range tests {
+			j := random.Intn(i + 1) //nolint:gosec // Not cryptographic.
+			tests[i], tests[j] = tests[j], tests[i]
+		}
+	}
+
+	// Assign tests to our shard.
+	paths := []string{}
+	names := []string{}
+	for idx, test := range tests {
+		if idx%n != shard {
+			continue
+		}
+		paths = append(paths, "./"+test.path)
+		names = append(names, test.name)
+	}
+
+	// De-dupe.
+	slices.Sort(paths)
+	slices.Sort(names)
+	paths = slices.Compact(paths)
+	names = slices.Compact(names)
+
+	fmt.Printf("-run ^(%s)$ %s\n", strings.Join(names, "|"), strings.Join(paths, " "))
+}