Add support for Python 3.13

Author: potiuk

.github/actions/post_tests_success/action.yml

@@ -47,7 +47,9 @@ runs:
       uses: codecov/codecov-action@v4
       env:
         CODECOV_TOKEN: ${{ inputs.codecov-token }}
-      if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12'
+      if: >
+        env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12'
+        && inputs.python-version != '3.13'
       with:
         name: coverage-${{env.JOB_ID}}
         flags: python-${{ env.PYTHON_MAJOR_MINOR_VERSION }},${{ env.BACKEND }}-${{ env.BACKEND_VERSION }}

.github/actions/prepare_all_ci_images/action.yml

@@ -66,3 +66,9 @@ runs:
         platform: ${{ inputs.platform }}
         python: "3.12"
         python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.13"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.13"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}

Dockerfile

@@ -111,6 +111,8 @@ else
     exit 1
 fi
 
+PYTHON_MAJOR_MINOR_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}', end='')")
+
 function get_dev_apt_deps() {
     if [[ "${DEV_APT_DEPS=}" == "" ]]; then
         DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \
@@ -182,6 +184,12 @@ function install_debian_dev_dependencies() {
     echo
     # shellcheck disable=SC2086
     apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}
+    # TODO(potiuk) remove me: until we got pendulum released with Python 3.13 support, we have to
+    # install latest cargo to support let..else statement from pendulum
+    if [[ "${PYTHON_MAJOR_MINOR_VERSION}" == "3.13" ]]; then
+        echo "Installing cargo"
+        curl https://sh.rustup.rs -sSf | sh -s -- -y
+    fi
 }
 
 function install_debian_runtime_dependencies() {
@@ -202,6 +210,12 @@ function install_debian_runtime_dependencies() {
     apt-get autoremove -yqq --purge
     apt-get clean
     rm -rf /var/lib/apt/lists/* /var/log/*
+    # TODO(potiuk) remove me: until we got pendulum released with Python 3.13 support, we have to
+    # install latest cargo to support let..else statement from pendulum
+    if [[ "${PYTHON_MAJOR_MINOR_VERSION}" == "3.13" ]]; then
+        echo "Installing cargo"
+        curl https://sh.rustup.rs -sSf | sh -s -- -y
+    fi
 }
 
 if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
@@ -1459,7 +1473,7 @@ COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scrip
 RUN bash /scripts/docker/install_mysql.sh dev && \
     bash /scripts/docker/install_mssql.sh dev && \
     bash /scripts/docker/install_postgres.sh dev
-ENV PATH=${PATH}:/opt/mssql-tools/bin
+ENV PATH=${HOME}/.cargo/bin:${PATH}:/opt/mssql-tools/bin
 
 # By default we do not install from docker context files but if we decide to install from docker context
 # files, we should override those variables to "docker-context-files"

Dockerfile.ci

@@ -50,6 +50,8 @@ else
     exit 1
 fi
 
+PYTHON_MAJOR_MINOR_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}', end='')")
+
 function get_dev_apt_deps() {
     if [[ "${DEV_APT_DEPS=}" == "" ]]; then
         DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \
@@ -121,6 +123,12 @@ function install_debian_dev_dependencies() {
     echo
     # shellcheck disable=SC2086
     apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}
+    # TODO(potiuk) remove me: until we got pendulum released with Python 3.13 support, we have to
+    # install latest cargo to support let..else statement from pendulum
+    if [[ "${PYTHON_MAJOR_MINOR_VERSION}" == "3.13" ]]; then
+        echo "Installing cargo"
+        curl https://sh.rustup.rs -sSf | sh -s -- -y
+    fi
 }
 
 function install_debian_runtime_dependencies() {
@@ -141,6 +149,12 @@ function install_debian_runtime_dependencies() {
     apt-get autoremove -yqq --purge
     apt-get clean
     rm -rf /var/lib/apt/lists/* /var/log/*
+    # TODO(potiuk) remove me: until we got pendulum released with Python 3.13 support, we have to
+    # install latest cargo to support let..else statement from pendulum
+    if [[ "${PYTHON_MAJOR_MINOR_VERSION}" == "3.13" ]]; then
+        echo "Installing cargo"
+        curl https://sh.rustup.rs -sSf | sh -s -- -y
+    fi
 }
 
 if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
@@ -1357,8 +1371,8 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     UV_LINK_MODE=copy \
     AIRFLOW_PRE_COMMIT_VERSION=${AIRFLOW_PRE_COMMIT_VERSION}
 
-# The PATH is needed for PIPX to find the tools installed
-ENV PATH="/root/.local/bin:${PATH}"
+# The PATH is needed for PIPX to find the tools installed and cargo to build the wheels
+ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}"
 
 # Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from
 # an incorrect architecture.

README.md

@@ -96,14 +96,14 @@ Airflow is not a streaming solution, but it is often used to process real-time d
 
 Apache Airflow is tested with:
 
-|            | Main version (dev)     | Stable version (2.10.5)    |
-|------------|------------------------|----------------------------|
-| Python     | 3.9, 3.10, 3.11, 3.12  | 3.8, 3.9, 3.10, 3.11, 3.12 |
-| Platform   | AMD64/ARM64(\*)        | AMD64/ARM64(\*)            |
-| Kubernetes | 1.29, 1.30, 1.31, 1.32 | 1.27, 1.28, 1.29, 1.30     |
-| PostgreSQL | 13, 14, 15, 16, 17     | 12, 13, 14, 15, 16         |
-| MySQL      | 8.0, 8.4, Innovation   | 8.0, 8.4, Innovation       |
-| SQLite     | 3.15.0+                | 3.15.0+                    |
+|            | Main version (dev)          | Stable version (2.10.5)    |
+|------------|-----------------------------|----------------------------|
+| Python     | 3.9, 3.10, 3.11, 3.12, 3.13 | 3.8, 3.9, 3.10, 3.11, 3.12 |
+| Platform   | AMD64/ARM64(\*)             | AMD64/ARM64(\*)            |
+| Kubernetes | 1.29, 1.30, 1.31, 1.32      | 1.27, 1.28, 1.29, 1.30     |
+| PostgreSQL | 13, 14, 15, 16, 17          | 12, 13, 14, 15, 16         |
+| MySQL      | 8.0, 8.4, Innovation        | 8.0, 8.4, Innovation       |
+| SQLite     | 3.15.0+                     | 3.15.0+                    |
 
 \* Experimental
 

airflow-core/docs/installation/prerequisites.rst

@@ -20,7 +20,7 @@ Prerequisites
 
 Airflow® is tested with:
 
-* Python: 3.9, 3.10, 3.11, 3.12
+* Python: 3.9, 3.10, 3.11, 3.12, 3.13
 
 * Databases:
 

airflow-core/docs/start.rst

@@ -24,7 +24,7 @@ This quick start guide will help you bootstrap an Airflow standalone instance on
 
 .. note::
 
-   Successful installation requires a Python 3 environment. Starting with Airflow 2.7.0, Airflow supports Python 3.9, 3.10, 3.11, and 3.12.
+   Successful installation requires a Python 3 environment. Starting with Airflow 2.7.0, Airflow supports Python 3.9, 3.10, 3.11, 3.12, 3.13.
 
    Officially supported installation methods include ``pip`` and ``uv``. Both tools provide a streamlined workflow for installing Airflow and managing dependencies.
 

airflow-core/pyproject.toml

@@ -35,7 +35,7 @@ name = "apache-airflow-core"
 description = "Core packages for Apache Airflow, schedule and API server"
 readme = { file = "README.md", content-type = "text/markdown" }
 license-files.globs = ["LICENSE", "3rd-party-licenses/*.txt", "NOTICE"]
-requires-python = "~=3.9,<3.13"
+requires-python = "~=3.9,<3.14"
 authors = [
     { name = "Apache Software Foundation", email = "[email protected]" },
 ]
@@ -106,6 +106,7 @@ dependencies = [
     "pathspec>=0.9.0",
     'pendulum>=2.1.2,<4.0;python_version<"3.12"',
     'pendulum>=3.0.0,<4.0;python_version>="3.12"',
+    'pendulum @ git+https://github.com/python-pendulum/pendulum.git ; python_version>="3.13"',
     "pluggy>=1.5.0",
     "psutil>=5.8.0",
     "pydantic>=2.11.0",
@@ -194,10 +195,8 @@ dependencies = [
 ]
 
 "pandas" = [
-    # In pandas 2.2 minimal version of the sqlalchemy is 2.0
-    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
-    # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
-    "pandas>=2.1.2,<2.3",
+    'pandas>=2.1.2; python_version <"3.13"',
+    'pandas>=2.2.0; python_version >="3.13"',
 ]
 "rabbitmq" = [
     "amqp>=5.2.0",
@@ -239,7 +238,6 @@ Mastodon = "https://fosstodon.org/@airflow"
 Bluesky = "https://bsky.app/profile/apache-airflow.bsky.social"
 YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
 
-
 [tool.hatch.version]
 path = "src/airflow/__init__.py"
 
@@ -275,6 +273,9 @@ exclude = [
     "src/airflow/api_fastapi/auth/managers/simple/ui/node_modules",
 ]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [dependency-groups]
 dev = [
     "apache-airflow-core[all]",

airflow-core/src/airflow/cli/commands/dag_command.py

@@ -646,6 +646,7 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No
     logical_date = args.logical_date or timezone.utcnow()
     use_executor = args.use_executor
 
+    # TODO(potiuk): we should likely get rid of rege passed by user here
     mark_success_pattern = (
         re.compile(args.mark_success_pattern) if args.mark_success_pattern is not None else None
     )

airflow-core/src/airflow/metrics/validators.py

@@ -253,6 +253,7 @@ def _has_pattern_match(self, name: str) -> bool:
         return False
 
 
+# TODO(potiuk): check if regexp pattern matching in list validations are safe
 class PatternAllowListValidator(ListValidator):
     """Match the provided strings anywhere in the metric name."""
 
@@ -263,6 +264,7 @@ def test(self, name: str) -> bool:
             return True  # default is all metrics are allowed
 
 
+# TODO(potiuk): check if regexp pattern matching in list validations are safe
 class PatternBlockListValidator(ListValidator):
     """Only allow names that do not match the blocked strings."""