From 75a73687db068005c6b33f79986fb7225241ec98 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Mon, 12 Aug 2024 14:17:58 -0700 Subject: [PATCH] parallel sdk builds --- .github/actions/setup-tools/action.yml | 9 ++- .github/workflows/run-acceptance-tests.yml | 68 ++++--------------- .gitignore | 4 ++ .mise.toml | 79 +++++++++++++++++++++- Makefile | 2 +- 5 files changed, 103 insertions(+), 59 deletions(-) diff --git a/.github/actions/setup-tools/action.yml b/.github/actions/setup-tools/action.yml index 8b502b7..0723692 100644 --- a/.github/actions/setup-tools/action.yml +++ b/.github/actions/setup-tools/action.yml @@ -4,6 +4,11 @@ description: Installs all tools necessary for building and testing the project. runs: using: "composite" steps: + - name: Collect workflow stats + uses: catchpoint/workflow-telemetry-action@v2 + with: + comment_on_pr: false + - name: Setup mise uses: jdx/mise-action@v2 @@ -11,8 +16,8 @@ runs: id: go-cache-paths shell: sh run: | - echo "::set-output name=go-build::$(go env GOCACHE)" - echo "::set-output name=go-mod::$(go env GOMODCACHE)" + echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT - name: Go build cache uses: actions/cache@v2 diff --git a/.github/workflows/run-acceptance-tests.yml b/.github/workflows/run-acceptance-tests.yml index add81a6..df0d576 100644 --- a/.github/workflows/run-acceptance-tests.yml +++ b/.github/workflows/run-acceptance-tests.yml @@ -63,6 +63,8 @@ jobs: body: "Please view the PR build: ${{ steps.vars.outputs.run-url }}" if: github.event_name == 'repository_dispatch' prerequisites: + if: github.event_name == 'repository_dispatch' || + github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest name: prerequisites steps: @@ -143,50 +145,9 @@ jobs: author_name: Failure in building provider prerequisites fields: repo,commit,author,action status: ${{ job.status }} - if: github.event_name == 'repository_dispatch' || - github.event.pull_request.head.repo.full_name == github.repository - build_sdks: - needs: prerequisites - runs-on: pulumi-ubuntu-8core - strategy: - fail-fast: true - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - name: build_sdks - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - with: - lfs: true - ref: ${{ env.PR_COMMIT_SHA }} - - name: Setup tools - uses: ./.github/actions/setup-tools - - id: version - name: Set Provider Version - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Download provider + tfgen binaries - uses: actions/download-artifact@v4 - with: - name: pulumi-${{ env.PROVIDER }}-provider.tar.gz - path: ${{ github.workspace }}/bin - - name: UnTar provider binaries - run: tar -zxf ${{ github.workspace }}/bin/provider.tar.gz -C ${{ - github.workspace}}/bin - - name: Restore Binary Permissions - run: - find ${{ github.workspace }} -name "pulumi-*-${{ env.PROVIDER }}" -print - -exec chmod +x {} \; - - name: Generate SDK - run: make generate_${{ matrix.language }} - - name: Build SDK - run: make build_${{ matrix.language }} + + - name: Generate & Build SDK + run: mise run sdk - name: Check worktree clean uses: pulumi/git-status-check-action@v1 with: @@ -198,12 +159,12 @@ jobs: sdk/python/pyproject.toml - run: git status --porcelain - name: Tar SDK folder - run: tar -zcf sdk/${{ matrix.language }}.tar.gz -C sdk/${{ matrix.language }} . + run: tar -zcf sdk.tar.gz -C sdk . - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ matrix.language }}-sdk.tar.gz - path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz + name: sdk.tar.gz + path: ${{ github.workspace}}/sdk.tar.gz retention-days: 30 - if: failure() && github.event_name == 'push' name: Notify Slack @@ -212,12 +173,11 @@ jobs: author_name: Failure while building SDKs fields: repo,commit,author,action status: ${{ job.status }} - if: github.event_name == 'repository_dispatch' || - github.event.pull_request.head.repo.full_name == github.repository + test: runs-on: pulumi-ubuntu-8core needs: - - build_sdks + - prerequisites strategy: fail-fast: true matrix: @@ -259,12 +219,10 @@ jobs: - name: Download SDK uses: actions/download-artifact@v4 with: - name: ${{ matrix.language }}-sdk.tar.gz - path: ${{ github.workspace}}/sdk/ + name: sdk.tar.gz + path: ${{ github.workspace}} - name: UnTar SDK folder - run: - tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{ - github.workspace}}/sdk/${{ matrix.language}} + run: tar -zxf ${{ github.workspace}}/sdk.tar.gz -C ${{github.workspace}}/sdk - name: Update path run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - name: Install Node dependencies diff --git a/.gitignore b/.gitignore index 728e7ee..dbd55b8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ sdk/dotnet/version.txt sdk/java/.gradle sdk/java/build/ sdk/java/build.gradle + +# Allow local mise overrides. +.mise.*.local.toml +.mise.local.toml diff --git a/.mise.toml b/.mise.toml index 08b4426..dcb0508 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,12 +1,89 @@ +[tasks.provider] +run = "make provider" +sources = ["provider/**/*.go", "go.mod"] +outputs = ["bin/pulumi-resource-docker-build"] + +# Generate + +[tasks."generate:nodejs"] +depends = ["provider"] +run = "make generate_nodejs" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["sdk/nodejs/**/*.ts"] + +[tasks."generate:python"] +run = "make generate_python" +depends = ["provider"] +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["sdk/python/**/*.py"] + +[tasks."generate:dotnet"] +depends = ["provider"] +run = "make generate_dotnet" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["sdk/dotnet/**/*.cs"] + +[tasks."generate:go"] +depends = ["provider"] +run = "make generate_go" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["sdk/go/**/*.go"] + +[tasks."generate:java"] +depends = ["provider"] +run = "make generate_java" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["sdk/java/**/*.java"] + +# Build + +[tasks."build:nodejs"] +depends = ["generate:nodejs"] +run = "make build_nodejs" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["*"] + +[tasks."build:python"] +depends = ["generate:python"] +run = "make build_python" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["*"] + +[tasks."build:dotnet"] +depends = ["generate:dotnet"] +run = "make build_dotnet" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["*"] + +[tasks."build:go"] +depends = ["generate:go"] +run = "make build_go" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["*"] + +[tasks."build:java"] +depends = ["generate:java"] +run = "make build_java" +sources = ["bin/pulumi-resource-docker-build"] +outputs = ["*"] + +[tasks.sdk] +depends = ["generate:*", "build:*"] + + [tools] +# Development tooling. go = "latest" # Will read version from go.mod. +lefthook = "1.7.12" +golangci-lint = "1.58.1" + +# Build and test tools. pulumi = "3.128.0" node = "22" dotnet = "6" python = "3.11.9" java = "temurin-11" gradle = "7.6" -golangci-lint = "1.58.1" "go:github.com/pulumi/schema-tools" = "v0.6.0" "go:github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt" = "v2.5.0" "go:github.com/pulumi/pulumictl/cmd/pulumictl" = "a1b89aaf4299fe5bc78e140fc60eba79088b3dc5" # blampe/replace branch doesn't work? diff --git a/Makefile b/Makefile index 89d3ca1..2f3bbc9 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ bin/${PROVIDER}: $(shell find ./provider -name '*.go') go.mod bin/pulumi-gen-${PACK}: # Required by CI touch bin/pulumi-gen-${PACK} -go.mod: $(shell find . -name '*.go') +go.mod: $(shell find provider -name '*.go') go.sum: go.mod sdk: $(shell mkdir -p sdk)