Execute integration tests on BitBar

.github/workflows/bitbar-prepare-artifacts.yaml

@@ -0,0 +1,126 @@
+name: Prepare device farm artifacts
+on:
+  workflow_call:
+    secrets:
+      # 1. Open the keychain on your mac and export the signing certificate and private key in a Certificates.p12 file
+      # 2. Convert your certificate to Base64 string: `base64 Certificates.p12 | pbcopy`
+      # 3. Update the value of the CERTIFICATES_FILE_BASE64 action secret with the content of the clipboard
+      # 4. Update the CERTIFICATES_PASSWORD action secret with the one used during the export of the certificate from the keychain
+      CERTIFICATES_FILE_BASE64:
+        description: 'Apple signing certificate'
+        required: true
+
+      # The password for your Apple signing certificate
+      CERTIFICATES_PASSWORD:
+        description: 'Apple signing certificate p12 password'
+        required: true
+
+      # A new keychain will be created during the run. The password could be any new random string.
+      KEYCHAIN_PASSWORD:
+        description: 'Keychain password'
+        required: true
+
+      # 1. Find the 'com.pingidentity.PingTestHost' provisioning profile (~/Library/MobileDevice/Provisioning\ Profiles)
+      # 2. Rename the file to `provisioning_profile.mobileprovision`
+      # 3. Zip the file: `zip provisioning_profile.mobileprovision.zip provisioning_profile.mobileprovision`
+      # 4. Convert the file to Base64 string: `base64 -i provisioning_profile.mobileprovision.zip | pbcopy`
+      # 5. Update the value of the BUILD_PROVISION_PROFILE_ZIP_BASE64 action secret with the content of the clipboard
+      BUILD_PROVISION_PROFILE:
+        description: 'Apple build provisioning profile'
+        required: true
+
+      SLACK_WEBHOOK:
+        description: 'Slack Notifier Incoming Webhook URL'
+        required: true
+jobs:
+  prepare-device-farm-artifacts:
+    runs-on: macos-14
+
+    steps:
+      # Clone the repo
+      - name: Clone the repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+          fetch-depth: 0
+
+      # Install the Apple certificate and provisioning profile
+      - name: Install the Apple certificate and provisioning profile
+        run: |
+          # Create variables
+          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
+          PP_PATH_ZIP=$RUNNER_TEMP/build_pp.mobileprovision.zip
+          PP_FILENAME=provisioning_profile.mobileprovision
+          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
+
+          # Import certificate and provisioning profile from secrets
+          echo -n ${{ secrets.CERTIFICATES_FILE_BASE64 }} | base64 --decode -o $CERTIFICATE_PATH
+          echo -n ${{ secrets.BUILD_PROVISION_PROFILE }} | base64 --decode -o $PP_PATH_ZIP
+          unzip $PP_PATH_ZIP
+
+          # Create temporary keychain
+          security create-keychain -p "${{ secrets.KEYCHAIN_PASSWORD }}" $KEYCHAIN_PATH
+          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
+          security unlock-keychain -p "${{ secrets.KEYCHAIN_PASSWORD }}" $KEYCHAIN_PATH
+
+          # Import certificate to keychain
+          security import $CERTIFICATE_PATH -P "${{ secrets.CERTIFICATES_PASSWORD }}" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
+          security list-keychain -d user -s $KEYCHAIN_PATH
+
+          # Apply provisioning profile
+          mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
+          cp $PP_FILENAME ~/Library/MobileDevice/Provisioning\ Profiles
+
+      # Set target Xcode version. For more details and options see:
+      # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11-Readme.md
+      - name: Select Xcode
+        run: sudo xcode-select -switch /Applications/Xcode_15.4.app && /usr/bin/xcodebuild -version
+
+      - name: build-for-testing and sign
+        run: xcodebuild -scheme PingTestHost -sdk iphoneos17.5 -workspace SampleApps/Ping.xcworkspace -configuration Debug clean build BUILD_DIR=/tmp/build/ DEVELOPMENT_TEAM=9QSE66762D -allowProvisioningUpdates -destination generic/platform=iOS -derivedDataPath /tmp/build/derivedData/ build-for-testing
+
+      # Prepare BitBar artifacts:
+      - name: Prepare BitBar artifacts
+        run: |
+          pwd
+          ls -la
+          cd /tmp/build/Debug-iphoneos/
+          cp -r PingTestHost.app/PlugIns/DavinciTests.xctest .
+          zip -r -X DavinciTests.xctest.zip DavinciTests.xctest
+          mkdir  Payload
+          cp -r PingTestHost.app Payload/
+          zip --symlinks -qr PingTestHost.ipa Payload
+
+      # Publish e2e tests and app build artifacts
+      - name: Publish PingTestHost.ipa
+        uses: actions/upload-artifact@v4
+        if: success()
+        with:
+          name: PingTestHost.ipa
+          path: /tmp/build/Debug-iphoneos/PingTestHost.ipa
+
+      - name: Publish DavinciTests.xctest.zip
+        uses: actions/upload-artifact@v4
+        if: success()
+        with:
+          name: DavinciTests.xctest.zip
+          path: /tmp/build/Debug-iphoneos/DavinciTests.xctest.zip
+
+      # Send slack notification ONLY if any of the steps above fail
+      - name: Send slack notification
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: all
+          custom_payload: |
+            {
+              attachments: [{
+                title: ':no_entry: Failed to prepare BitBar test artifacts',
+                color: 'danger',
+                text: `\nWorkflow: ${process.env.AS_WORKFLOW} -> ${process.env.AS_JOB}\nPull request: ${process.env.AS_PULL_REQUEST}\nCommit: ${process.env.AS_COMMIT} by ${process.env.AS_AUTHOR}`, 
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+        if: failure()

.github/workflows/bitbar-results.yaml

@@ -0,0 +1,100 @@
+name: Wait for BitBar Test Run Results
+
+on:
+  workflow_call:
+    inputs:
+      bitbar-project-id:
+        description: BitBar Project ID
+        type: string
+        default: ${{ vars.BITBAR_PROJECT_ID }}
+
+      bitbar-run-id:
+        description: BitBar Run ID
+        type: string
+        required: true
+
+    outputs:
+      bitbar-run-url:
+        description: "The BitBar run URL"
+        value: ${{ jobs.bitbar-run.outputs.bitbar_run_url }}
+
+    secrets:
+      BITBAR_API_KEY:
+        description: BitBar API Key
+        required: true
+      SLACK_WEBHOOK:
+        description: Slack Notifier Incoming Webhook
+        required: true
+
+jobs:
+  bitbar-results:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Workflow inputs:"
+        run: |
+          echo "Project ID - ${{ inputs.bitbar-project-id }}"
+          echo "Run ID - ${{ inputs.bitbar-run-id }}"
+
+      - name: Wait for BitBar test run to finish...
+        timeout-minutes: 30
+        run: |
+          (
+            until [ "$(curl -s -u ${{ secrets.BITBAR_API_KEY }}: https://cloud.bitbar.com/api/me/projects/${{ inputs.bitbar-project-id }}/runs/${{ inputs.bitbar-run-id }} | jq -r '.state')" == "FINISHED" ];
+            do
+              echo "Waiting for BitBar Results. Sleeping for 10 seconds..."
+              sleep 10
+            done
+          )
+          echo "BITBAR_TEST_RUN_RESULT=$(curl -s -u ${{ secrets.BITBAR_API_KEY }}: https://cloud.bitbar.com/api/me/projects/${{ inputs.bitbar-project-id }}/runs/${{ inputs.bitbar-run-id }})" >> $GITHUB_ENV
+
+      # Get the outcome json of the test run.
+      - name: Parse test run outcome json
+        run: |
+          echo ${{ env.BITBAR_TEST_RUN_RESULT }}
+          echo "==========================================="
+          echo "projectName: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.projectName')"
+          echo "displayName: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.displayName')"
+          echo "executedTestCaseCount: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.executedTestCaseCount')"
+          echo "successfulTestCaseCount: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.successfulTestCaseCount')"
+          echo "failedTestCaseCount: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.failedTestCaseCount')"
+          echo "runningDeviceCount: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.runningDeviceCount')"
+          echo "totalDeviceCount: $(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq '.totalDeviceCount')"
+          echo "==========================================="
+          echo "BITBAR_PROJECT_NAME=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.projectName')" >> $GITHUB_ENV
+          echo "BITBAR_RUN_DISPLAY_NAME=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.displayName')" >> $GITHUB_ENV
+          echo "BITBAR_RUN_NUMBER=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.number')" >> $GITHUB_ENV
+          echo "BITBAR_EXECUTED_TESTS_COUNT=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.executedTestCaseCount')" >> $GITHUB_ENV
+          echo "BITBAR_SUCCESS_TESTS_COUNT=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.successfulTestCaseCount')" >> $GITHUB_ENV
+          echo "BITBAR_FAILED_TESTS_COUNT=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.failedTestCaseCount')" >> $GITHUB_ENV
+          echo "BITBAR_SUCCESS_RATIO=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.successRatio')" >> $GITHUB_ENV          
+          echo "BITBAR_DEVICE_COUNT=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.deviceCount')" >> $GITHUB_ENV
+          echo "BITBAR_DEVICE_GROUP_NAME=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.config.usedDeviceGroupName')" >> $GITHUB_ENV
+          echo "BITBAR_TEST_RUN_URL=$(echo '${{ env.BITBAR_TEST_RUN_RESULT }}' | jq -r '.uiLink')" >> $GITHUB_ENV
+
+      # Check for failures and set the outcome of the workflow
+      - name: Set job status
+        run: |
+          if [[ ${{env.BITBAR_FAILED_TESTS_COUNT}} != '0' ]]; then
+            exit 1
+          else
+            exit 0
+          fi
+
+      # Send slack notification with result status
+      - name: Send slack notification
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: all
+          custom_payload: |
+            {
+              attachments: [{
+                title: 'BitBar ${{ env.BITBAR_PROJECT_NAME }} - #${{ env.BITBAR_RUN_NUMBER }}',
+                title_link: '${{ env.BITBAR_TEST_RUN_URL }}',
+                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+                text: `\nTest summary: ${{ job.status }} in ${process.env.AS_TOOK}\nPassed: ${{ env.BITBAR_SUCCESS_TESTS_COUNT }}, Failed: ${{ env.BITBAR_FAILED_TESTS_COUNT }}\nDevice group: ${{ env.BITBAR_DEVICE_GROUP_NAME }}, Number of devices: ${{ env.BITBAR_DEVICE_COUNT }}\n\nWorkflow: ${process.env.AS_WORKFLOW} -> ${process.env.AS_JOB}\nPull request: ${process.env.AS_PULL_REQUEST}\nCommit: ${process.env.AS_COMMIT} by ${process.env.AS_AUTHOR}\nMessage: ${process.env.AS_MESSAGE}`, 
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+        if: always()

.github/workflows/bitbar-run.yaml

@@ -0,0 +1,115 @@
+name: Run tests in BitBar Cloud
+on:
+  workflow_call:
+    inputs:
+      bitbar-project-id:
+        description: BitBar project id
+        type: string
+        default: ${{ vars.BITBAR_PROJECT_ID }} 
+
+      bitbar-device-group-id:
+        description: The device group id to run tests against
+        type: string
+        default: ${{ vars.BITBAR_DEVICE_GROUP_ID }}
+
+      bitbar-os-type: 
+        description: OS Type
+        type: string
+        default: IOS
+
+      bitbar-framework-id:
+        description: The framework id
+        type: string
+        default: ${{ vars.BITBAR_FRAMEWORK_ID }}
+
+    outputs:
+      bitbar-run-id:
+        description: The newly created run id in BitBar
+        value: ${{ jobs.bitbar-run.outputs.bitbar_run_id }}
+
+    secrets:
+      # To obtain a new API key: https://cloud.bitbar.com/#user/security-center
+      BITBAR_API_KEY:
+        description: BitBar API Key
+        required: true
+
+      SLACK_WEBHOOK:
+        description: 'Slack Notifier Incoming Webhook URL'
+        required: true
+jobs:
+  bitbar-run:
+    runs-on: ubuntu-latest
+    outputs:
+      bitbar_run_id: ${{ steps.bitbar_run_id.outputs.bitbar_run_id }}
+
+    steps:
+      # Get the test artifacts prepared in previous step
+      - name: Get PingTestHost.ipa BitBar artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: PingTestHost.ipa
+
+      - name: Get the DavinciTests.xctest.zip BitBar artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: DavinciTests.xctest.zip
+
+      - name: Unzip PingTestHost.ipa and DavinciTests.xctest.zip
+        run: |
+          unzip -o PingTestHost.ipa
+          unzip -o DavinciTests.xctest.zip
+
+      - name: Upload PingTestHost.ipa to BitBar
+        run: |
+          echo "BITBAR_APP_FILE_ID=$(curl -X POST -u ${{ secrets.BITBAR_API_KEY }}: https://cloud.bitbar.com/api/me/files -F "[email protected]" | jq '.id')" >> $GITHUB_ENV
+
+      - name: Upload DavinciTests.xctest.zip to BitBar
+        run: |
+          echo "BITBAR_TEST_FILE_ID=$(curl -X POST -u ${{ secrets.BITBAR_API_KEY }}: https://cloud.bitbar.com/api/me/files -F "[email protected]" | jq '.id')"  >> $GITHUB_ENV
+
+      - name: Prepare BitBar run configuration file
+        run: |
+          (
+            echo "{"
+            echo "\"osType\":\"${{ inputs.bitbar-os-type }}\","
+            echo "\"projectId\":${{ inputs.bitbar-project-id }},"
+            echo "\"frameworkId\":${{ inputs.bitbar-framework-id }},"
+            echo "\"deviceGroupId\":${{ inputs.bitbar-device-group-id }},"
+            echo "\"files\":["
+            echo "    {\"id\":${{ env.BITBAR_APP_FILE_ID }}, \"action\": \"INSTALL\"},"
+            echo "    {\"id\":${{ env.BITBAR_TEST_FILE_ID }}, \"action\": \"RUN_TEST\"}"
+            echo "]"
+            echo "}"
+          ) > bitbar-run-configuration.txt
+
+      - name: Display bitbar-run-configuration.txt
+        run: |
+          cat bitbar-run-configuration.txt
+
+      # Start the test run
+      - name: Start a test run
+        run: |
+          echo "BITBAR_TEST_RUN_ID=$(curl -H 'Content-Type: application/json' -u ${{ secrets.BITBAR_API_KEY }}: https://cloud.bitbar.com/api/me/runs --data-binary @bitbar-run-configuration.txt | jq '.id')"  >> $GITHUB_ENV
+
+      # Set bitbar_run_id as output of the workflow. This is needed for the next workflow to continue
+      - name: Set the bitbar_run_id output
+        id: bitbar_run_id
+        run: echo "::set-output name=bitbar_run_id::${{ env.BITBAR_TEST_RUN_ID }}"
+
+      # Send slack notification ONLY if any of the steps above fail
+      - name: Send slack notification
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: all
+          custom_payload: |
+            {
+              attachments: [{
+                title: ':no_entry: Failed to start BitBar test run!',
+                color: 'danger',
+                text: `\nWorkflow: ${process.env.AS_WORKFLOW} -> ${process.env.AS_JOB}\nPull request: ${process.env.AS_PULL_REQUEST}\nCommit: ${process.env.AS_COMMIT} by ${process.env.AS_AUTHOR}`, 
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+        if: failure()

.github/workflows/ci.yaml

@@ -25,3 +25,35 @@ jobs:
       MEND_EMAIL: ${{ secrets.MEND_EMAIL }}
       MEND_USER_KEY: ${{ secrets.MEND_USER_KEY }}
       SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} 
+
+  # Build and sign BitBar test artifacts (PingTestHost.ipa and DavinciTests.xctest.zip)
+  bitbar-prepare-artifacts:
+    name: Prepare device farm artifacts
+    uses: ./.github/workflows/bitbar-prepare-artifacts.yaml
+    needs: build-and-test
+    secrets:
+      CERTIFICATES_FILE_BASE64: ${{ secrets.CERTIFICATES_FILE_BASE64 }}
+      CERTIFICATES_PASSWORD: ${{ secrets.CERTIFICATES_PASSWORD }}
+      KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+      BUILD_PROVISION_PROFILE: ${{ secrets.BUILD_PROVISION_PROFILE_ZIP_BASE64}}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+  # Execute e2e test cases in BitBar. The workflow outputs the newly created run id.
+  bitbar-run:
+    name: Run e2e tests in BitBar
+    uses: ./.github/workflows/bitbar-run.yaml
+    needs: bitbar-prepare-artifacts
+    secrets:
+      BITBAR_API_KEY: ${{ secrets.BITBAR_API_KEY }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+  # Wait for BitBar test run to finish and publish results
+  bitbar-results:
+    name: Wait for and publish BitBar test results
+    uses: ./.github/workflows/bitbar-results.yaml
+    needs: bitbar-run
+    secrets:
+      BITBAR_API_KEY: ${{ secrets.BITBAR_API_KEY }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+    with:
+      bitbar-run-id: ${{ needs.bitbar-run.outputs.bitbar-run-id }}