# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Tests
on:  # yamllint disable-line rule:truthy
  schedule:
    - cron: '28 0 * * *'
  push:
    branches: ['main', 'v[0-9]+-[0-9]+-test']
  pull_request:
    branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable']
permissions:
  # All other permissions are set to none
  contents: read
  packages: read
env:
  ANSWER: "yes"
  DB_RESET: "true"
  VERBOSE: "true"
  GITHUB_REPOSITORY: ${{ github.repository }}
  GITHUB_USERNAME: ${{ github.actor }}
  # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
  # Airflow one is going to be used
  CONSTRAINTS_GITHUB_REPOSITORY: >-
    ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
  # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}"
  IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
  USE_SUDO: "true"
  INCLUDE_SUCCESS_OUTPUTS: "true"

concurrency:
  group: ci-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:

  build-info:
    name: "Build info"
    # The runs-on cannot refer to env. or secrets. context, so we have no
    # option but to specify a hard-coded list here. This is "safe", as the list
    # is checked again by the runner using it's own list, so a PR author cannot
    # change this and get access to our self-hosted runners
    #
    # This list is kept up-to-date from the list of authors found in the
    # 'airflow-ci-infra' by the 'sync_authors' Github workflow. It uses a regexp
    # to find the list of authors and replace them, so any changes to the
    # formatting of the contains(fromJSON()) structure below will need to be
    # reflected in that workflow too.
    runs-on: >-
      ${{ (
        (
          github.event_name == 'push' ||
          github.event_name == 'schedule' ||
          contains(fromJSON('[
            "BasPH",
            "Fokko",
            "KevinYang21",
            "Taragolis",
            "XD-DENG",
            "aijamalnk",
            "alexvanboxel",
            "aoen",
            "artwr",
            "ashb",
            "bbovenzi",
            "bolkedebruin",
            "criccomini",
            "dimberman",
            "dstandish",
            "eladkal",
            "ephraimbuddy",
            "feluelle",
            "feng-tao",
            "houqp",
            "jedcunningham",
            "jgao54",
            "jghoman",
            "jhtimmins",
            "jmcarp",
            "josh-fell",
            "kaxil",
            "leahecole",
            "malthe",
            "mik-laj",
            "milton0825",
            "mistercrunch",
            "msumit",
            "o-nikolas",
            "pierrejeambrun",
            "pingzh",
            "potiuk",
            "r39132",
            "ryanahamilton",
            "ryw",
            "saguziel",
            "sekikn",
            "turbaszek",
            "uranusjr",
            "vikramkoka",
            "xinbinhuang",
            "yuqian90",
            "zhongjiajie"
          ]'), github.event.pull_request.user.login)
        ) && github.repository == 'apache/airflow'
      ) && 'self-hosted' || 'ubuntu-20.04' }}
    env:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
      upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
      python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
      all-python-versions-list-as-string: >-
        ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
      default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
      kubernetes-versions-list-as-string: >-
        ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
      kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
      default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
      postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
      default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
      mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
      mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }}
      default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
      default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
      default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
      full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
      test-types: ${{ steps.selective-checks.outputs.test-types }}
      postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
      mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
      mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
      sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
      providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
      skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
      run-tests: ${{ steps.selective-checks.outputs.run-tests }}
      run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
      run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
      basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
      image-build: ${{ steps.selective-checks.outputs.image-build }}
      docs-build: ${{ steps.selective-checks.outputs.docs-build }}
      needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
      needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
      needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
      default-branch: ${{ steps.selective-checks.outputs.default-branch }}
      default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
      docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
      skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
      debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
      source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
      pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
      in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
      build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
      runs-on: ${{ steps.source-run-info.outputs.runs-on }}
      canary-run: ${{ steps.source-run-info.outputs.canary-run }}
      run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          submodules: recursive
      - name: Fetch incoming commit ${{ github.sha }} with its parent
        uses: actions/checkout@v3
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: "Retrieve defaults from branch_defaults.py"
        id: defaults
        # We could retrieve it differently here - by just importing the variables and
        # printing them from python code, however we want to have the same code as used in
        # the build-images.yml (there we cannot import python code coming from the PR - we need to
        # treat the python code as text and extract the variables from there.
        run: |
          python - <<EOF >> ${GITHUB_ENV}
          from pathlib import Path
          import re
          import sys

          DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
          BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
          CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'

          branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
          constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)

          output = f"""
          DEFAULT_BRANCH={branch}
          DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
          """.strip()

          print(output)
          # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
          print(output, file=sys.stderr)
          EOF
      - name: "Get information about the Workflow"
        id: source-run-info
        run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
      - name: Selective checks
        id: selective-checks
        env:
          PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
          COMMIT_REF: "${{ github.sha }}"
          VERBOSE: "false"
        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
      - name: env
        run: printenv
        env:
          PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
          GITHUB_CONTEXT: ${{ toJson(github) }}

  # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
  # tests to complete - it is run very early in the build process for "main" merges in order to refresh
  # cache using the current constraints. This will speed up cache refresh in cases when setup.py
  # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will
  # delay cache refresh. It does not attempt to upgrade to newer dependencies.
  # We only push CI cache as PROD cache usually does not gain as much from fresh cache because
  # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time
  push-early-buildx-cache-to-github-registry:
    permissions:
      packages: write
    timeout-minutes: 50
    name: "Push Early Image Cache"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
    strategy:
      fail-fast: false
      matrix:
        platform: ["linux/amd64", "linux/arm64"]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      UPGRADE_TO_NEWER_DEPENDENCIES: false
    continue-on-error: true
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
      - name: "Start ARM instance"
        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
        if: >
          matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
          && needs.build-info.outputs.default-branch == 'main'
      - name: "Push CI cache ${{ matrix.platform }}"
        run: >
          breeze ci-image build
          --builder airflow_cache
          --prepare-buildx-cache
          --run-in-parallel
          --force-build
          --platform ${{ matrix.platform }}
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
      - name: "Push CI latest image ${{ matrix.platform }}"
        run: >
          breeze ci-image build
          --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
        # We only push "amd" image as it is really only needed for any kind of automated builds in CI
        # and currently there is not an easy way to make multi-platform image from two separate builds
        if: >
          matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
          && needs.build-info.outputs.default-branch == 'main'
      - name: "Stop ARM instance"
        run: ./scripts/ci/images/ci_stop_arm_instance.sh
        if: >
          always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
          && needs.build-info.outputs.default-branch == 'main'
      - name: "Clean docker cache for ${{ matrix.platform }}"
        run: docker system prune --all --force
        if: >
          matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
          && needs.build-info.outputs.default-branch == 'main'
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: >
          always() && needs.build-info.outputs.canary-run == 'true'
          && needs.build-info.outputs.default-branch == 'main'
  # Check that after earlier cache push, breeze command will build quickly
  check-that-image-builds-quickly:
    timeout-minutes: 5
    name: "Check that image builds quickly"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
      - push-early-buildx-cache-to-github-registry
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      UPGRADE_TO_NEWER_DEPENDENCIES: false
      PLATFORM: "linux/amd64"
    if: needs.build-info.outputs.canary-run == 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: "Check that image builds quickly"
        run: breeze shell --max-time 120
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  build-ci-images:
    permissions:
      packages: write
    timeout-minutes: 80
    name: >-
      ${{needs.build-info.outputs.build-job-description}} CI images
      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: needs.build-info.outputs.in-workflow-build == 'true'
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.targetCommitSha }}
          persist-credentials: false
          submodules: recursive
        if: needs.build-info.outputs.in-workflow-build == 'true'
      - name: >
          Build CI Images
          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/build-ci-images
        if: needs.build-info.outputs.in-workflow-build == 'true'
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}

  build-prod-images:
    permissions:
      packages: write
    timeout-minutes: 80
    name: >
      ${{needs.build-info.outputs.build-job-description}} PROD images
      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, build-ci-images]
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      BACKEND: sqlite
      DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
      VERSION_SUFFIX_FOR_PYPI: "dev0"
      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: needs.build-info.outputs.in-workflow-build == 'true'
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.targetCommitSha }}
          persist-credentials: false
          submodules: recursive
        if: needs.build-info.outputs.in-workflow-build == 'true'
      - name: >
          Build PROD Images
          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/build-prod-images
        if: needs.build-info.outputs.in-workflow-build == 'true'
        with:
          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}

  run-breeze-tests:
    timeout-minutes: 10
    name: Breeze unit tests
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          persist-credentials: false
      - uses: actions/setup-python@v4
        with:
          python-version: "${{needs.build-info.outputs.default-python-version}}"
          cache: 'pip'
          cache-dependency-path: ./dev/breeze/setup*
      - run: python -m pip install --editable ./dev/breeze/
      - run: breeze setup version
      - run: python -m pytest ./dev/breeze/ -n auto --color=yes
      - run: breeze setup check-all-params-in-groups

  tests-www:
    timeout-minutes: 10
    name: React WWW tests
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    if: needs.build-info.outputs.run-www-tests == 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Setup node"
        uses: actions/setup-node@v3
        with:
          node-version: 14
      - name: "Cache eslint"
        uses: actions/cache@v3
        with:
          path: 'airflow/www/node_modules'
          key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
      - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
      - run: yarn --cwd airflow/www/ run test
        env:
          FORCE_COLOR: 2


  test-openapi-client-generation:
    timeout-minutes: 10
    name: "Test OpenAPI client generation"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    if: needs.build-info.outputs.needs-api-codegen == 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: "Generate client codegen diff"
        run: ./scripts/ci/openapi/client_codegen_diff.sh

  test-examples-of-prod-image-building:
    timeout-minutes: 60
    name: "Test examples of production image building"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    if: needs.build-info.outputs.image-build == 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: "Setup python"
        uses: actions/setup-python@v4
        with:
          python-version: "${{needs.build-info.outputs.default-python-version}}"
          cache: 'pip'
          cache-dependency-path: ./dev/requirements.txt
      - name: "Test examples of PROD image building"
        run: >
          python -m pip install -r ./docker_tests/requirements.txt &&
          python -m pytest docker_tests/test_examples_of_prod_image_building.py -n auto --color=yes

  test-git-clone-on-windows:
    timeout-minutes: 5
    name: "Test git clone on Windows"
    runs-on: windows-latest
    needs: [build-info]
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
          persist-credentials: false
    if: needs.build-info.outputs.runs-on != 'self-hosted'

  wait-for-ci-images:
    timeout-minutes: 120
    name: "Wait for CI images"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, build-ci-images]
    if: needs.build-info.outputs.image-build == 'true'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      BACKEND: sqlite
      # Force more parallelism for pull even on public images
      PARALLELISM: 6
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
        id: wait-for-images
        run: breeze ci-image pull --run-in-parallel --verify --wait-for-image --tag-as-latest
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  static-checks:
    timeout-minutes: 30
    name: "Static checks"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
    if: needs.build-info.outputs.basic-checks-only == 'false'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
        id: breeze
      - name: Cache pre-commit envs
        uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          # yamllint disable-line rule:line-length
          key: "pre-commit-full-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
          restore-keys: |
            pre-commit-full-${{steps.breeze.outputs.host-python-version}}
            pre-commit-full
      - name: "Static checks"
        run: breeze static-checks --all-files --show-diff-on-failure --color always
        env:
          VERBOSE: "false"
          SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
          COLUMNS: "250"
          SKIP_GROUP_OUTPUT: "true"
          DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
          RUFF_FORMAT: "github"
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  # Those checks are run if no image needs to be built for checks. This is for simple changes that
  # Do not touch any of the python code or any of the important files that might require building
  # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
  static-checks-basic-checks-only:
    timeout-minutes: 30
    name: "Static checks: basic checks only"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
    if: needs.build-info.outputs.basic-checks-only == 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Setup python"
        uses: actions/setup-python@v4
        with:
          python-version: "${{needs.build-info.outputs.default-python-version}}"
          cache: 'pip'
          cache-dependency-path: ./dev/breeze/setup*
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        id: breeze
      - name: Cache pre-commit envs
        uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          # yamllint disable-line rule:line-length
          key: "pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
          restore-keys: "\
            pre-commit-full-${{steps.breeze.outputs.host-python-version}}-\
            ${{ hashFiles('.pre-commit-config.yaml') }}\n
            pre-commit-basic-${{steps.breeze.outputs.host-python-version}}\n
            pre-commit-full-${{steps.breeze.outputs.host-python-version}}\n
            pre-commit-basic-\n
            pre-commit-full-"
      - name: Fetch incoming commit ${{ github.sha }} with its parent
        uses: actions/checkout@v3
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
          persist-credentials: false
      - name: "Static checks: basic checks only"
        run: >
          breeze static-checks --all-files --show-diff-on-failure --color always
          --commit-ref "${{ github.sha }}"
        env:
          VERBOSE: "false"
          SKIP_IMAGE_PRE_COMMITS: "true"
          SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
          COLUMNS: "250"
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  docs:
    timeout-minutes: 45
    name: "Build docs"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    if: needs.build-info.outputs.docs-build == 'true'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          submodules: recursive
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
      - uses: actions/cache@v3
        id: cache-doc-inventories
        with:
          path: ./docs/_inventory_cache/
          key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
          restore-keys: |
            docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
            docs-inventory-
      - name: "Build docs"
        run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }}
      - name: Configure AWS credentials
        uses: ./.github/actions/configure-aws-credentials
        if: >
          github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
          github.event_name == 'push'
        with:
          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-central-1
      - name: "Upload documentation to AWS S3"
        if: >
          github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
          github.event_name == 'push'
        run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  prepare-test-provider-packages:
    timeout-minutes: 80
    name: "Provider packages ${{matrix.package-format}}"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    strategy:
      matrix:
        package-format: ["sdist", "wheel"]
        exclude: "${{fromJson(needs.build-info.outputs.providers-package-format-exclude)}}"
      fail-fast: false
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      PACKAGE_FORMAT: "${{matrix.package-format}}"
      USE_AIRFLOW_VERSION: "${{matrix.package-format}}"
    if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Cleanup dist files"
        run: rm -fv ./dist/*
      - name: "Prepare provider documentation"
        run: breeze release-management prepare-provider-documentation
        if: matrix.package-format == 'wheel'
      - name: "Prepare provider packages: ${{matrix.package-format}}"
        run: breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0
      - name: "Prepare airflow package: ${{matrix.package-format}}"
        run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
      - name: "Verify wheel packages with twine"
        run: pipx install twine && twine check dist/*.whl
        if: matrix.package-format == 'wheel'
      - name: "Verify sdist packages with twine"
        run: pipx install twine && twine check dist/*.tar.gz
        if: matrix.package-format == 'sdist'
      - name: "Test providers issue generation automatically"
        run: >
          breeze release-management generate-issue-content-providers
          --only-available-in-dist --disable-progress
        if: matrix.package-format == 'wheel'
      - name: "Install and test provider packages and airflow via ${{matrix.package-format}} files"
        run: breeze release-management verify-provider-packages --use-packages-from-dist
        env:
          SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
      - name: "Remove airflow package and replace providers with 2.3-compliant versions"
        run: |
          rm -vf dist/apache_airflow-*.whl dist/apache_airflow_providers_docker*.whl
          pip download --no-deps --dest dist apache-airflow-providers-docker==3.1.0
        if: matrix.package-format == 'wheel'
      - name: "Get all provider extras as AIRFLOW_EXTRAS env variable"
        run: >
          python -c 'from pathlib import Path; import json;
          providers = json.loads(Path("generated/provider_dependencies.json").read_text());
          provider_keys = ",".join(providers.keys());
          print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV
        if: matrix.package-format == 'wheel'
      - name: "Install and test provider packages and airflow on Airflow 2.3 files"
        run: >
          breeze release-management verify-provider-packages --use-airflow-version 2.3.0
          --use-packages-from-dist --airflow-constraints-reference constraints-2.3.0
        if: matrix.package-format == 'wheel'
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  test-airflow-release-commands:
    timeout-minutes: 80
    name: "Test Airflow release commands"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    if: needs.build-info.outputs.runs-on == 'self-hosted'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Cleanup dist files"
        run: rm -fv ./dist/*
      - name: "Install required packages"
        run: python -m pip install --editable ./dev/breeze/
      - name: "Check Airflow create minor branch command"
        run: breeze release-management create-minor-branch --version-branch 2-4 -a y
      - name: "Check Airflow RC process command"
        run: |
          breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2  -a y
      - name: "Check Airflow release process command"
        run: |
          breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  tests-helm:
    timeout-minutes: 80
    name: "Python unit tests for Helm chart"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "Helm"
      BACKEND: ""
      DB_RESET: "false"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      JOB_ID: "helm-tests"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
    if: >
      needs.build-info.outputs.needs-helm-tests == 'true' &&
      (github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
      needs.build-info.outputs.default-branch == 'main'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Helm Unit Tests"
        run: breeze testing helm-tests
      - name: "Post Helm Tests"
        uses: ./.github/actions/post_tests

  test-pytest-collection:
    timeout-minutes: 5
    name: "Test Pytest collection"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images]
    if: needs.build-info.outputs.image-build == 'true'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      BACKEND: sqlite
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
        id: wait-for-images
        run: breeze ci-image pull --wait-for-image --tag-as-latest
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
      - name: "Tests Pytest collection"
        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py"
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  tests-postgres:
    timeout-minutes: 130
    name: >
      Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
      ${{needs.build-info.outputs.test-types}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    strategy:
      matrix:
        python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
        postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}"
        exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}"
      fail-fast: false
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "postgres"
      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
      POSTGRES_VERSION: "${{matrix.postgres-version}}"
      BACKEND_VERSION: "${{matrix.postgres-version}}"
      JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/migration_tests
      - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        run: breeze testing tests --run-in-parallel
      - name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py" arm
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests

  tests-mysql:
    timeout-minutes: 130
    name: >
      MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    strategy:
      matrix:
        python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
        mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}"
        exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}"
      fail-fast: false
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "mysql"
      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
      MYSQL_VERSION: "${{matrix.mysql-version}}"
      BACKEND_VERSION: "${{matrix.mysql-version}}"
      JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/migration_tests
      - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        run: breeze testing tests --run-in-parallel
      - name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
        if: env.MYSQL_VERSION != '5.7'
        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py" arm
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests

  tests-mssql:
    timeout-minutes: 130
    name: >
      MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    strategy:
      matrix:
        python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
        mssql-version: "${{fromJson(needs.build-info.outputs.mssql-versions)}}"
        exclude: "${{fromJson(needs.build-info.outputs.mssql-exclude)}}"
      fail-fast: false
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "mssql"
      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
      MSSQL_VERSION: "${{matrix.mssql-version}}"
      BACKEND_VERSION: "${{matrix.mssql-version}}"
      JOB_ID: "mssql-${{matrix.mssql-version}}-${{matrix.python-version}}"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/migration_tests
      - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        run: breeze testing tests --run-in-parallel
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests

  tests-sqlite:
    timeout-minutes: 130
    name: >
      Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
        exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }}
      fail-fast: false
    if: needs.build-info.outputs.run-tests == 'true'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "sqlite"
      BACKEND_VERSION: ""
      JOB_ID: "sqlite-${{matrix.python-version}}"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/migration_tests
      - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        run: breeze testing tests --run-in-parallel
      - name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py" arm
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests

  tests-integration-postgres:
    timeout-minutes: 130
    name: Integration Tests Postgres
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "postgres"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
      BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
      JOB_ID: "integration"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
      SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
      - name: "Integration Tests Postgres: cassandra"
        run: |
          breeze testing integration-tests --integration cassandra
          breeze stop
        if: needs.build-info.outputs.runs-on != 'self-hosted'
      - name: "Integration Tests Postgres: mongo"
        run: |
          breeze testing integration-tests --integration mongo
          breeze stop
        if: needs.build-info.outputs.runs-on != 'self-hosted'
      - name: "Integration Tests Postgres: pinot"
        run: |
          breeze testing integration-tests --integration pinot
          breeze stop
        if: needs.build-info.outputs.runs-on != 'self-hosted'
      - name: "Integration Tests Postgres: celery"
        run: |
          breeze testing integration-tests --integration celery
          breeze stop
        if: needs.build-info.outputs.runs-on != 'self-hosted'
      - name: "Integration Tests Postgres: trino, kerberos"
        run: |
          breeze testing integration-tests --integration trino --integration kerberos
          breeze stop
        if: needs.build-info.outputs.runs-on != 'self-hosted'
      - name: "Integration Tests Postgres: all-testable"
        run: breeze testing integration-tests --integration all-testable
        if: needs.build-info.outputs.runs-on == 'self-hosted'
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests

  tests-integration-mysql:
    timeout-minutes: 130
    name: Integration Tests MySQL
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, test-pytest-collection]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "mysql"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
      BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
      JOB_ID: "integration"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
      SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: needs.build-info.outputs.runs-on == 'self-hosted'
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
        if: needs.build-info.outputs.runs-on == 'self-hosted'
      - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
        uses: ./.github/actions/prepare_breeze_and_image
        if: needs.build-info.outputs.runs-on == 'self-hosted'
      - name: "Integration Tests MySQL: all-testable"
        run: breeze testing integration-tests --integration all-testable
        if: needs.build-info.outputs.runs-on == 'self-hosted'
      - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
        uses: ./.github/actions/post_tests
        if: needs.build-info.outputs.runs-on == 'self-hosted'


  tests-quarantined:
    timeout-minutes: 60
    name: "Quarantined tests"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    continue-on-error: true
    needs: [build-info, test-pytest-collection]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      TEST_TYPES: "Quarantined"
      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
      BACKEND: "sqlite"
      BACKEND_VERSION: ""
      JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}"
      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
      - name: >
          Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined
        run: breeze testing tests --run-in-parallel || true
      - name: >
          Post Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined"
        uses: ./.github/actions/post_tests

  summarize-warnings:
    timeout-minutes: 15
    name: "Summarize warnings"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
      - tests-postgres
      - tests-sqlite
      - tests-mysql
      - tests-mssql
      - tests-quarantined
      - tests-integration-postgres
      - tests-integration-mysql
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          submodules: recursive
      - name: "Download all artifacts from the current build"
        uses: actions/download-artifact@v3
        with:
          path: ./artifacts
      - name: "Summarize all warnings"
        run: |
          ls -R ./artifacts/
          cat ./artifacts/test-warnings*/* | sort | uniq
          echo
          echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)


  wait-for-prod-images:
    timeout-minutes: 120
    name: "Wait for PROD images"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-ci-images, build-prod-images]
    if: needs.build-info.outputs.image-build == 'true'
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      BACKEND: sqlite
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      # Force more parallelism for pull even on public images
      PARALLELISM: 6
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
        # We wait for the images to be available either from "build-images.yml' run as pull_request_target
        # or from build-prod-images above.
        # We are utilising single job to wait for all images because this job merely waits
        # For the images to be available and test them.
        run: breeze prod-image pull --verify --wait-for-image --run-in-parallel
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  test-docker-compose-quick-start:
    timeout-minutes: 60
    name: "Test docker-compose quick start"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-prod-images]
    if: needs.build-info.outputs.image-build == 'true'
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: >
          Prepare breeze & PROD image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          pull-image-type: 'PROD'
      - name: "Test docker-compose quick start"
        run: breeze testing docker-compose-tests
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  tests-kubernetes:
    timeout-minutes: 240
    name: "Helm: ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs: [build-info, wait-for-prod-images]
    strategy:
      matrix:
        executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
      fail-fast: false
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
    if: >
      ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
      needs.build-info.outputs.needs-helm-tests == 'true' ) &&
      needs.build-info.outputs.default-branch == 'main'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
        run: breeze prod-image pull --run-in-parallel --tag-as-latest
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
      - name: "Cache bin folder with tools for kubernetes testing"
        uses: actions/cache@v3
        with:
          path: ".build/.k8s-env"
          key: "\
            k8s-env-${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\
            'setup.py','pyproject.toml','generated/provider_dependencies.json') }}"
      - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos}}
        run: breeze k8s run-complete-tests --run-in-parallel --upgrade
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
          EXECUTOR: ${{matrix.executor}}
          VERBOSE: false
      - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos}}
        uses: actions/upload-artifact@v3
        if: failure() || cancelled()
        with:
          name: kind-logs-${{matrix.executor}}
          path: /tmp/kind_logs_*
          retention-days: 7
      - name: Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos}}
        uses: actions/upload-artifact@v3
        if: failure() || cancelled()
        with:
          name: k8s-test-resources-${{matrix.executor}}
          path: /tmp/k8s_test_resources_*
          retention-days: 7
      - name: "Delete clusters just in case they are left"
        run: breeze k8s delete-cluster --all
        if: always()
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  constraints:
    permissions:
      contents: write
    timeout-minutes: 80
    name: "Constraints"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
      - docs
      - wait-for-ci-images
      - wait-for-prod-images
      - static-checks
      - tests-sqlite
      - tests-mysql
      - tests-mssql
      - tests-postgres
      - tests-integration-postgres
      - tests-integration-mysql
      - push-early-buildx-cache-to-github-registry
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
    if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          submodules: recursive
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
        run: breeze ci-image pull --run-in-parallel --tag-as-latest
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
      - name: "Generate constraints"
        run: |
          breeze release-management generate-constraints --run-in-parallel \
              --airflow-constraints-mode constraints-source-providers
          breeze release-management generate-constraints \
              --run-in-parallel --airflow-constraints-mode constraints-no-providers
          breeze release-management generate-constraints \
              --run-in-parallel --airflow-constraints-mode constraints
        env:
          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
      - name: "Set constraints branch name"
        id: constraints-branch
        run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
        if: needs.build-info.outputs.canary-run == 'true'
      - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
        uses: actions/checkout@v3
        if: needs.build-info.outputs.canary-run == 'true'
        with:
          path: "repo"
          ref: ${{ steps.constraints-branch.outputs.branch }}
          persist-credentials: false
      - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
        run: ./scripts/ci/constraints/ci_commit_constraints.sh
        if: needs.build-info.outputs.canary-run == 'true'
      - name: "Push changes"
        uses: ./.github/actions/github-push-action
        if: needs.build-info.outputs.canary-run == 'true'
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ steps.constraints-branch.outputs.branch }}
          directory: "repo"
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
  # is executed as result of direct push to "main" or one of the "vX-Y-test" branches
  # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry
  # It will automatically check if a new python image was released and will pull the latest one if needed
  push-buildx-cache-to-github-registry:
    permissions:
      packages: write
    timeout-minutes: 50
    name: "Push Image Cache"
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
      - constraints
      - docs
    if: needs.build-info.outputs.canary-run == 'true'
    strategy:
      fail-fast: false
      matrix:
        platform: ["linux/amd64", "linux/arm64"]
    env:
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: >
          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
        uses: ./.github/actions/prepare_breeze_and_image
        env:
          # Always use default Python version of CI image for preparing packages
          PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
      - name: "Cleanup dist and context file"
        run: rm -fv ./dist/* ./docker-context-files/*
      - name: "Prepare airflow package for PROD build"
        run: breeze release-management prepare-airflow-package --package-format wheel
        env:
          VERSION_SUFFIX_FOR_PYPI: "dev0"
      - name: "Prepare providers packages for PROD build"
        run: >
          breeze release-management prepare-provider-packages
          --package-list-file ./scripts/ci/installed_providers.txt
          --package-format wheel
        env:
          VERSION_SUFFIX_FOR_PYPI: "dev0"
        if: needs.build-info.outputs.default-branch == 'main'
      - name: "Start ARM instance"
        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
        if: matrix.platform == 'linux/arm64'
      - name: "Push CI cache ${{ matrix.platform }}"
        run: >
          breeze ci-image build
          --builder airflow_cache
          --prepare-buildx-cache
          --run-in-parallel
          --force-build
          --platform ${{ matrix.platform }}
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
      - name: "Push CI latest image ${{ matrix.platform }}"
        run: >
          breeze ci-image build
          --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
        if: matrix.platform == 'linux/amd64'
      - name: "Move dist packages to docker-context files"
        run: mv -v ./dist/*.whl ./docker-context-files
      - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
        run: >
          breeze prod-image build
          --builder airflow_cache
          --install-packages-from-context
          --prepare-buildx-cache
          --platform ${{ matrix.platform }}
      - name: "Push PROD latest image ${{ matrix.platform }}"
        run: >
          breeze prod-image build --tag-as-latest --install-packages-from-context
          --push --run-in-parallel --platform ${{ matrix.platform }}
        env:
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
        if: matrix.platform == 'linux/amd64'
      - name: "Stop ARM instance"
        run: ./scripts/ci/images/ci_stop_arm_instance.sh
        if: always() && matrix.platform == 'linux/arm64'
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()

  build-ci-arm-images:
    timeout-minutes: 50
    name: >
      Build CI ARM images
      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
    runs-on: "${{needs.build-info.outputs.runs-on}}"
    needs:
      - build-info
      - wait-for-ci-images
      - wait-for-prod-images
      - static-checks
      - tests-sqlite
      - tests-mysql
      - tests-mssql
      - tests-postgres
      - tests-integration-postgres
      - tests-integration-mysql
    env:
      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
    if: >
      needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
      needs.build-info.outputs.in-workflow-build == 'true' &&
      needs.build-info.outputs.canary-run != 'true'
    steps:
      - name: Cleanup repo
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.build-info.outputs.targetCommitSha }}
          persist-credentials: false
          submodules: recursive
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
      - name: "Start ARM instance"
        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
      - name: >
          Build CI ARM images ${{ env.IMAGE_TAG }}
          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
        run: >
          breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
        env:
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
      - name: "Stop ARM instance"
        run: ./scripts/ci/images/ci_stop_arm_instance.sh
        if: always()
      - name: "Fix ownership"
        run: breeze ci fix-ownership
        if: always()