parallel sdk builds

2024-08-12 14:17:58 -07:00
parent f2b29d43ec
commit 75a73687db
5 changed files with 103 additions and 59 deletions
--- a/.github/actions/setup-tools/action.yml
+++ b/.github/actions/setup-tools/action.yml
@@ -4,6 +4,11 @@ description: Installs all tools necessary for building and testing the project.
 runs:
  using: "composite"
  steps:
+    - name: Collect workflow stats
+      uses: catchpoint/workflow-telemetry-action@v2
+      with:
+        comment_on_pr: false
+
    - name: Setup mise
      uses: jdx/mise-action@v2

@@ -11,8 +16,8 @@ runs:
      id: go-cache-paths
      shell: sh
      run: |
-        echo "::set-output name=go-build::$(go env GOCACHE)"
-        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+        echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+        echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT

    - name: Go build cache
      uses: actions/cache@v2
--- a/.github/workflows/run-acceptance-tests.yml
+++ b/.github/workflows/run-acceptance-tests.yml
@@ -63,6 +63,8 @@ jobs:
          body: "Please view the PR build: ${{ steps.vars.outputs.run-url }}"
    if: github.event_name == 'repository_dispatch'
  prerequisites:
+    if: github.event_name == 'repository_dispatch' ||
+      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    name: prerequisites
    steps:
@@ -143,50 +145,9 @@ jobs:
          author_name: Failure in building provider prerequisites
          fields: repo,commit,author,action
          status: ${{ job.status }}
-    if: github.event_name == 'repository_dispatch' ||
-      github.event.pull_request.head.repo.full_name == github.repository
-  build_sdks:
-    needs: prerequisites
-    runs-on: pulumi-ubuntu-8core
-    strategy:
-      fail-fast: true
-      matrix:
-        language:
-          - nodejs
-          - python
-          - dotnet
-          - go
-          - java
-    name: build_sdks
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-          ref: ${{ env.PR_COMMIT_SHA }}
-      - name: Setup tools
-        uses: ./.github/actions/setup-tools
-      - id: version
-        name: Set Provider Version
-        uses: pulumi/provider-version-action@v1
-        with:
-          set-env: PROVIDER_VERSION
-      - name: Download provider + tfgen binaries
-        uses: actions/download-artifact@v4
-        with:
-          name: pulumi-${{ env.PROVIDER }}-provider.tar.gz
-          path: ${{ github.workspace }}/bin
-      - name: UnTar provider binaries
-        run: tar -zxf ${{ github.workspace }}/bin/provider.tar.gz -C ${{
-          github.workspace}}/bin
-      - name: Restore Binary Permissions
-        run:
-          find ${{ github.workspace }} -name "pulumi-*-${{ env.PROVIDER }}" -print
-          -exec chmod +x {} \;
-      - name: Generate SDK
-        run: make generate_${{ matrix.language }}
-      - name: Build SDK
-        run: make build_${{ matrix.language }}
+
+      - name: Generate & Build SDK
+        run: mise run sdk
      - name: Check worktree clean
        uses: pulumi/git-status-check-action@v1
        with:
@@ -198,12 +159,12 @@ jobs:
            sdk/python/pyproject.toml
      - run: git status --porcelain
      - name: Tar SDK folder
-        run: tar -zcf sdk/${{ matrix.language }}.tar.gz -C sdk/${{ matrix.language }} .
+        run: tar -zcf sdk.tar.gz -C sdk .
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
-          name: ${{ matrix.language  }}-sdk.tar.gz
-          path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
+          name: sdk.tar.gz
+          path: ${{ github.workspace}}/sdk.tar.gz
          retention-days: 30
      - if: failure() && github.event_name == 'push'
        name: Notify Slack
@@ -212,12 +173,11 @@ jobs:
          author_name: Failure while building SDKs
          fields: repo,commit,author,action
          status: ${{ job.status }}
-    if: github.event_name == 'repository_dispatch' ||
-      github.event.pull_request.head.repo.full_name == github.repository
+
  test:
    runs-on: pulumi-ubuntu-8core
    needs:
-      - build_sdks
+      - prerequisites
    strategy:
      fail-fast: true
      matrix:
@@ -259,12 +219,10 @@ jobs:
      - name: Download SDK
        uses: actions/download-artifact@v4
        with:
-          name: ${{ matrix.language }}-sdk.tar.gz
-          path: ${{ github.workspace}}/sdk/
+          name: sdk.tar.gz
+          path: ${{ github.workspace}}
      - name: UnTar SDK folder
-        run:
-          tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{
-          github.workspace}}/sdk/${{ matrix.language}}
+        run: tar -zxf ${{ github.workspace}}/sdk.tar.gz -C ${{github.workspace}}/sdk
      - name: Update path
        run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
      - name: Install Node dependencies
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,7 @@ sdk/dotnet/version.txt
 sdk/java/.gradle
 sdk/java/build/
 sdk/java/build.gradle
+
+# Allow local mise overrides.
+.mise.*.local.toml
+.mise.local.toml
--- a/.mise.toml
+++ b/.mise.toml
@@ -1,12 +1,89 @@
+[tasks.provider]
+run = "make provider"
+sources = ["provider/**/*.go", "go.mod"]
+outputs = ["bin/pulumi-resource-docker-build"]
+
+# Generate
+
+[tasks."generate:nodejs"]
+depends = ["provider"]
+run = "make generate_nodejs"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["sdk/nodejs/**/*.ts"]
+
+[tasks."generate:python"]
+run = "make generate_python"
+depends = ["provider"]
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["sdk/python/**/*.py"]
+
+[tasks."generate:dotnet"]
+depends = ["provider"]
+run = "make generate_dotnet"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["sdk/dotnet/**/*.cs"]
+
+[tasks."generate:go"]
+depends = ["provider"]
+run = "make generate_go"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["sdk/go/**/*.go"]
+
+[tasks."generate:java"]
+depends = ["provider"]
+run = "make generate_java"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["sdk/java/**/*.java"]
+
+# Build
+
+[tasks."build:nodejs"]
+depends = ["generate:nodejs"]
+run = "make build_nodejs"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["*"]
+
+[tasks."build:python"]
+depends = ["generate:python"]
+run = "make build_python"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["*"]
+
+[tasks."build:dotnet"]
+depends = ["generate:dotnet"]
+run = "make build_dotnet"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["*"]
+
+[tasks."build:go"]
+depends = ["generate:go"]
+run = "make build_go"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["*"]
+
+[tasks."build:java"]
+depends = ["generate:java"]
+run = "make build_java"
+sources = ["bin/pulumi-resource-docker-build"]
+outputs = ["*"]
+
+[tasks.sdk]
+depends = ["generate:*", "build:*"]
+
+
 [tools]
+# Development tooling.
 go = "latest" # Will read version from go.mod.
+lefthook = "1.7.12"
+golangci-lint = "1.58.1"
+
+# Build and test tools.
 pulumi = "3.128.0"
 node = "22"
 dotnet = "6"
 python = "3.11.9"
 java = "temurin-11"
 gradle = "7.6"
-golangci-lint = "1.58.1"
 "go:github.com/pulumi/schema-tools" = "v0.6.0"
 "go:github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt" = "v2.5.0"
 "go:github.com/pulumi/pulumictl/cmd/pulumictl" = "a1b89aaf4299fe5bc78e140fc60eba79088b3dc5" # blampe/replace branch doesn't work?
--- a/2
+++ b/2
@@ -180,7 +180,7 @@ bin/${PROVIDER}: $(shell find ./provider -name '*.go') go.mod
 bin/pulumi-gen-${PACK}: # Required by CI
 	touch bin/pulumi-gen-${PACK}

-go.mod: $(shell find . -name '*.go')
+go.mod: $(shell find provider -name '*.go')
 go.sum: go.mod

 sdk: $(shell mkdir -p sdk)