Merge branch 'apache:main' into master

.github/boring-cyborg.yml

@@ -252,3 +252,4 @@ checkUpToDate:
     - airflow/migrations/*
     - airflow/migrations/**/*
     - airflow/alembic.ini
+    - airflow/**/*.py

.github/workflows/ci.yml

@@ -179,6 +179,7 @@ jobs:
       pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
       pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
       runsOn: ${{ steps.set-runs-on.outputs.runsOn }}
+      runCoverage: ${{ steps.set-run-coverage.outputs.runCoverage }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -235,6 +236,15 @@ jobs:
             echo "Maintainer or main run running with self-hosted runner"
             echo "::set-output name=runsOn::\"self-hosted\""
           fi
+      # Avoid having to specify the coverage logic every time.
+      - name: Set run coverage
+        id: set-run-coverage
+        run: |
+          echo "::set-output name=runCoverage::true"
+        if: >
+          github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
+          github.event_name == 'push' &&
+          steps.selective-checks.outputs.default-branch == 'main'
 
   run-new-breeze-tests:
     timeout-minutes: 10
@@ -748,6 +758,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: >
             coverage-helm
@@ -811,6 +822,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: >
             coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
@@ -873,6 +885,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
           path: "./files/coverage*.xml"
@@ -934,6 +947,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
           path: "./files/coverage*.xml"
@@ -993,6 +1007,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: coverage-sqlite-${{matrix.python-version}}
           path: ./files/coverage*.xml
@@ -1068,6 +1083,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           retention-days: 7
       - name: "Upload artifact for coverage"
         uses: actions/upload-artifact@v2
+        if: needs.build-info.outputs.runCoverage == 'true'
         with:
           name: coverage-quarantined-${{ matrix.backend }}
           path: "./files/coverage*.xml"
@@ -1088,10 +1104,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     env:
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
     # Only upload coverage on merges to main
-    if: >
-      github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
-      github.event_name == 'push' &&
-      needs.build-info.outputs.default-branch == 'main'
+    if: needs.build-info.outputs.runCoverage == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2

CONTRIBUTING.rst

@@ -582,7 +582,7 @@ Airflow dependencies
 
    Only ``pip`` installation is currently officially supported.
 
-   While they are some successes with using other tools like `poetry <https://python-poetry.org/>`_ or
+   While there are some successes with using other tools like `poetry <https://python-poetry.org/>`_ or
    `pip-tools <https://pypi.org/project/pip-tools/>`_, they do not share the same workflow as
    ``pip`` - especially when it comes to constraint vs. requirements management.
    Installing via ``Poetry`` or ``pip-tools`` is not currently supported.

Dockerfile

@@ -44,7 +44,7 @@ ARG AIRFLOW_HOME=/opt/airflow
 ARG AIRFLOW_UID="50000"
 ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 
-ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster"
 
 ARG AIRFLOW_PIP_VERSION=21.3.1
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -434,25 +434,28 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /var/log/*
 
-# Only copy install_m(y/s)sql. We do not need any other scripts in the final image.
+# Only copy mysql/mssql installation scripts for now - so that changing the other
+# scripts which are needed much later will not invalidate the docker layer here.
 COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/
-
-# fix permission issue in Azure DevOps when running the scripts
-RUN chmod a+x /scripts/docker/install_mysql.sh && \
-    /scripts/docker/install_mysql.sh prod && \
-    chmod a+x /scripts/docker/install_mssql.sh && \
-    /scripts/docker/install_mssql.sh && \
-    adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
-           --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \
+# We run chmod +x to fix permission issue in Azure DevOps when running the scripts
+# However when AUFS Docker backend is used, this might cause "text file busy" error
+# when script is executed right after it's executable flag has been changed, so
+# we run additional sync afterwards. See https://github.com/moby/moby/issues/13594
+RUN chmod a+x /scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh \
+    && sync \
+    && /scripts/docker/install_mysql.sh prod \
+    && /scripts/docker/install_mssql.sh \
+    && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
+           --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" \
 # Make Airflow files belong to the root group and are accessible. This is to accommodate the guidelines from
 # OpenShift https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html
-    mkdir -pv "${AIRFLOW_HOME}"; \
-    mkdir -pv "${AIRFLOW_HOME}/dags"; \
-    mkdir -pv "${AIRFLOW_HOME}/logs"; \
-    chown -R airflow:0 "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}"; \
-    chmod -R g+rw "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" ; \
-    find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x; \
-    find "${AIRFLOW_USER_HOME_DIR}" -executable -print0 | xargs --null chmod g+x
+    && mkdir -pv "${AIRFLOW_HOME}" \
+    && mkdir -pv "${AIRFLOW_HOME}/dags" \
+    && mkdir -pv "${AIRFLOW_HOME}/logs" \
+    && chown -R airflow:0 "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \
+    && chmod -R g+rw "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \
+    && find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x \
+    && find "${AIRFLOW_USER_HOME_DIR}" -executable -print0 | xargs --null chmod g+x
 
 COPY --chown=airflow:0 --from=airflow-build-image \
      "${AIRFLOW_USER_HOME_DIR}/.local" "${AIRFLOW_USER_HOME_DIR}/.local"
@@ -463,10 +466,10 @@ COPY --chown=airflow:0 scripts/in_container/prod/clean-logs.sh /clean-logs
 # See https://github.com/apache/airflow/issues/9248
 # Set default groups for airflow and root user
 
-RUN chmod a+x /entrypoint /clean-logs && \
-    chmod g=u /etc/passwd  && \
-    chmod g+w "${AIRFLOW_USER_HOME_DIR}/.local" && \
-    usermod -g 0 airflow -G 0
+RUN chmod a+x /entrypoint /clean-logs \
+    && chmod g=u /etc/passwd \
+    && chmod g+w "${AIRFLOW_USER_HOME_DIR}/.local" \
+    && usermod -g 0 airflow -G 0
 
 # make sure that the venv is activated for all users
 # including plain sudo, sudo with --interactive flag

Dockerfile.ci

@@ -15,14 +15,14 @@
 #
 # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
 #
-ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster"
 FROM ${PYTHON_BASE_IMAGE} as main
 
 # Nolog bash flag is currently ignored - but you can replace it with other flags (for example
 # xtrace - to show commands executed)
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 
-ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster"
 ARG AIRFLOW_VERSION="2.2.0.dev0"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
@@ -96,9 +96,17 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 # Only copy mysql/mssql installation scripts for now - so that changing the other
-# scripts which are needed much later will not invalidate the docker layer here
+# scripts which are needed much later will not invalidate the docker layer here.
 COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
-RUN /scripts/docker/install_mysql.sh dev && /scripts/docker/install_mssql.sh \
+# We run chmod +x to fix permission issue in Azure DevOps when running the scripts
+# However when AUFS Docker backend is used, this might cause "text file busy" error
+# when script is executed right after it's executable flag has been changed, so
+# we run additional sync afterwards. See https://github.com/moby/moby/issues/13594
+RUN chmod a+x /scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh \
+    && sync  \
+    && /scripts/docker/install_mysql.sh prod  \
+    && /scripts/docker/install_mysql.sh dev  \
+    && /scripts/docker/install_mssql.sh \
     && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
               --quiet "airflow" --home "/home/airflow" \
     && echo -e "airflow\nairflow" | passwd airflow 2>&1 \
@@ -260,10 +268,9 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
 # Those are additional constraints that are needed for some extras but we do not want to
 # force them on the main Airflow package. Those limitations are:
 # * certifi<2021.0.0: required by snowflake provider
-# * lazy-object-proxy<1.5.0: required by astroid
 # * dill<0.3.3 required by apache-beam
 # * google-ads<14.0.1 required to prevent updating google-python-api>=2.0.0
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="lazy-object-proxy<1.5.0 dill<0.3.3 certifi<2021.0.0 google-ads<14.0.1"
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0 google-ads<14.0.1"
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
     UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}

IMAGES.rst

@@ -372,7 +372,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | Build argument                           | Default value                            | Description                              |
 +==========================================+==========================================+==========================================+
-| ``PYTHON_BASE_IMAGE``                    | ``python:3.6-slim-buster``               | Base Python image                        |
+| ``PYTHON_BASE_IMAGE``                    | ``python:3.7-slim-buster``               | Base Python image                        |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PYTHON_MAJOR_MINOR_VERSION``           | ``3.6``                                  | major/minor version of Python (should    |
 |                                          |                                          | match base image)                        |

INTHEWILD.md

@@ -44,6 +44,7 @@ Currently, **officially** using Airflow:
 1. [Akamas](https://akamas.io) [[@GiovanniPaoloGibilisco](https://github.com/GiovanniPaoloGibilisco), [@lucacavazzana](https://github.com/lucacavazzana)]
 1. [Alan](https://alan.eu) [[@charles-go](https://github.com/charles-go)]
 1. [allegro.pl](http://allegro.tech/) [[@kretes](https://github.com/kretes)]
+1. [Allo-Media](https://www.allo-media.net) [[@merwan](https://github.com/merwan/)]
 1. [AloPeyk](https://alopeyk.com) [[@blcksrx](https://github.com/blcksrx), [@AloPeyk](https://github.com/AloPeyk)]
 1. [Altafino](https://altafino.com) [[@altafino](https://github.com/altafino)]
 1. [AltX](https://www.getaltx.com/about) [[@pedromduarte](https://github.com/pedromduarte)]
@@ -161,6 +162,7 @@ Currently, **officially** using Airflow:
 1. [Drivy](https://www.drivy.com) [[@AntoineAugusti](https://github.com/AntoineAugusti)]
 1. [Dropbox](https://www.dropbox.com) [[@AlexeySanko](https://github.com/AlexeySanko)]
 1. [Dunnhumby](https://www.dunnhumby.com)
+1. [Dunzo](https://www.dunzo.com)[[@masterlittle](https://github.com/masterlittle)]
 1. [Dynata](https://www.dynata.com) [[@neil3handari](https://github.com/neil3handari)]
 1. [Easy Taxi](http://www.easytaxi.com/) [[@caique-lima](https://github.com/caique-lima) & [@diraol](https://github.com/diraol)]
 1. [EBANX](https://www.ebanx.com/) [[@estevammr](https://github.com/estevammr) & [@nathangngencissk](https://github.com/nathangngencissk) & [@raafaadg](https://github.com/raafaadg) & [@whrocha](https://github.com/whrocha)]
@@ -428,6 +430,7 @@ Currently, **officially** using Airflow:
 1. [Twitter](https://www.twitter.com/) [[@aoen](https://github.com/aoen)]
 1. [Ubisoft](https://www.ubisoft.com/) [[@Walkoss](https://github.com/Walkoss)]
 1. [Udacity](https://www.udacity.com/) [[@dandikunited](https://github.com/DandikUnited), [@simon-uc](https://github.com/simon-uc)]
+1. [Uh!ive](https://uh.live/) [[@merwan](https://github.com/merwan/)]
 1. [Umami Collective](https://umamicollective.com) [[@juanuicich](https://github.com/juanuicich)]
 1. [Unacast](https://unacast.com)[[@unacast](https://github.com/unacast)]
 1. [United Airlines](https://www.united.com/) [[@ilopezfr](https://github.com/ilopezfr)]