Merge branch 'tensorflow:master' into AlexanderLavelle/schemagen_mod_…

…for_sequences

.github/workflows/cd-docs.yml

@@ -0,0 +1,53 @@
+name: deploy-docs
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+  pull_request:
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+        if: (github.event_name != 'pull_request')
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+          cache-dependency-path: |
+            setup.py
+            tfx/dependencies.py
+            requirements-docs.txt
+
+      - name: Save time for cache for mkdocs
+        run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+
+      - name: Caching
+        uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+
+      - name: Install Dependencies
+        run: pip install -r requirements-docs.txt
+
+      - name: Deploy to GitHub Pages
+        run: mkdocs gh-deploy --force
+        if: (github.event_name != 'pull_request')
+
+      - name: Build docs to check for errors
+        run: mkdocs build
+        if: (github.event_name == 'pull_request')

.github/workflows/ci-lint.yml

@@ -0,0 +1,33 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+     branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+      with:
+        # Ensure the full history is fetched
+        # This is required to run pre-commit on a specific set of commits
+        # TODO: Remove this when all the pre-commit issues are fixed
+        fetch-depth: 0
+    - uses: actions/setup-python@v5.1.1
+      with:
+        python-version: 3.9
+    - name: Determine commit range
+      id: commit_range
+      run: |
+       echo "TO_REF=${{ github.sha }}" >> $GITHUB_ENV
+        if [ "${{ github.event_name }}" == "pull_request" ]; then
+          echo "FROM_REF=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV
+        else
+          echo "FROM_REF=${{ github.event.before }}" >> $GITHUB_ENV
+        fi
+    - uses: pre-commit/action@v3.0.1
+      with:
+        # TODO: Remove this when all the pre-commit issues are fixed
+        extra_args: --from-ref ${{ env.FROM_REF }} --to-ref ${{ env.TO_REF }}

.github/workflows/ci-test.yml

@@ -1,89 +1,77 @@
-# Github action definitions for ci-test with PRs.
+# Github action definitions for unit-tests with PRs.
 
-name: tfx-ci-test
+name: tfx-unit-tests
 on:
+  push:
   pull_request:
     branches: [ master ]
     paths-ignore:
       - '**.md'
       - 'docs/**'
+  workflow_dispatch:
+
+env:
+  USE_BAZEL_VERSION: "6.5.0"
+  # Changed to match tensorflow
+  # https://github.com/tensorflow/tensorflow/blob/master/.bazelversion
 
 jobs:
-  build:
+  tests:
     if: github.actor != 'copybara-service[bot]'
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10']
+        which-tests: ["not e2e", "e2e"]
+        dependency-selector: ["NIGHTLY", "DEFAULT"]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Get Changed Files
-      id: changed_files
-      uses: trilom/file-changes-action@v1.2.4
-      with:
-        fileOutput: ' '
-    - name: Select files to check
-      run: |
-        # Filter out non-python files.
-        (cat $HOME/files_added.txt; echo; cat $HOME/files_modified.txt) | tr ' ' '\n' | grep '\.py$' > py_files.txt || true
-        # Filter out non-test python files and e2e or integration tests.
-        cat py_files.txt | grep '_test\.py$' | grep -v _e2e_ | grep -v integration | grep -v 'examples/' > py_test_files.txt || true
-        # Select proto files.
-        (cat $HOME/files_added.txt; echo; cat $HOME/files_modified.txt) | tr ' ' '\n' | grep '\.proto$' > proto_files.txt || true
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
       with:
-        python-version: 3.9
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: false
+        docker-images: true
+        swap-storage: true
 
-    - name: Set up Bazel 5.3.0
-      run: |
-        # Instruction from https://docs.bazel.build/versions/master/install-ubuntu.html
-        curl -sSL https://github.com/bazelbuild/bazel/releases/download/5.3.0/bazel-5.3.0-installer-linux-x86_64.sh -o bazel_installer.sh
-        chmod +x bazel_installer.sh
-        sudo ./bazel_installer.sh
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.py
+          tfx/dependencies.py
 
-    - name: Cache pip
-      uses: actions/cache@v2
+    - name: Set up Bazel
+      uses: bazel-contrib/setup-bazel@0.8.5
       with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding setup.py + TFX version 
-        key: ${{ runner.os }}-pip-${{ hashFiles('tfx/dependencies.py') }}-
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        # Avoid downloading Bazel every time.
+        bazelisk-cache: true
+        # Store build cache per workflow.
+        disk-cache: ${{ github.workflow }}-${{ hashFiles('.github/workflows/ci-test.yml') }}
+        # Share repository cache between workflows.
+        repository-cache: true
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade pip wheel setuptools
         # TODO(b/232490018): Cython need to be installed separately to build pycocotools.
         python -m pip install Cython -c ./test_constraints.txt
-        TFX_DEPENDENCY_SELECTOR=NIGHTLY pip install -c ./test_constraints.txt --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre --editable .[all]
+        pip install \
+          -c ./${{ matrix.dependency-selector == 'NIGHTLY' && 'nightly_test_constraints.txt' || 'test_constraints.txt' }} \
+          --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre .[all]
 
-    - name: Run unit tests
-      shell: bash
-      run: |
-        [ ! -s "py_test_files.txt" ] || cat py_test_files.txt | xargs -I {} python {}
-
-    - name: Lint with protolint
-      continue-on-error: true
       env:
-        PROTOLINT_VERSION: 0.25.1
-      shell: bash
-      run: |
-        curl -sSOL https://github.com/yoheimuta/protolint/releases/download/v${PROTOLINT_VERSION}/protolint_${PROTOLINT_VERSION}_Linux_x86_64.tar.gz
-        tar zxf protolint_${PROTOLINT_VERSION}_Linux_x86_64.tar.gz
-        echo "[NOTE] This linter is currently EXPERIMENTAL.======================================="
-        echo "Please contact reviewers for existing lint errors or false negative errors."
-        echo "===================================================================================="
-        [ ! -s "proto_files.txt" ] || cat proto_files.txt | xargs -I {} ./protolint {}
+        TFX_DEPENDENCY_SELECTOR: ${{ matrix.dependency-selector }}
 
-    - name: Lint with pylint
-      continue-on-error: true
+    - name: Run unit tests
       shell: bash
       run: |
-        pip install pylint
-        echo "[NOTE] This linter is currently EXPERIMENTAL.======================================="
-        echo "Please contact reviewers for existing lint errors or false negative errors."
-        echo "Feel free to send PRs for pylintrc in the root directory of the repository if needed."
-        echo "===================================================================================="
-        [ ! -s "py_files.txt" ] || pylint $(cat py_files.txt | tr '\n' ' ')
+        pytest -m "${{ matrix.which-tests }}"

.github/workflows/wheels.yml

@@ -0,0 +1,130 @@
+name: Build Wheels & Publish to PyPI
+
+on:
+  pull_request:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+env:
+  USE_BAZEL_VERSION: "7.2.1"
+
+jobs:
+  build_sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repo
+      uses: actions/checkout@v4
+
+    - name: Set up python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: install python dependencies
+      run: pip install build twine
+
+    - name: build sdist
+      run: |
+        python -m build --sdist -o wheelhouse
+
+    - name: List and check sdist
+      run: |
+        ls -lh wheelhouse/
+        twine check wheelhouse/*
+
+    - name: Upload sdist
+      uses: actions/upload-artifact@v4
+      with:
+        name: sdist
+        path: ./wheelhouse/*.tar.gz
+
+  build_wheels:
+    name: >
+      build ${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }}
+      ${{ (matrix.arch) || '' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        python-version: ['cp39', 'cp310']
+
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    steps:
+    - name: Check out the repo
+      uses: actions/checkout@v4
+
+    - name: Set up python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Install python build dependencies
+      run: |
+          pip install wheel
+
+    - uses: bazel-contrib/setup-bazel@0.8.5
+      name: Set up Bazel
+      with:
+        # Avoid downloading Bazel every time.
+        bazelisk-cache: true
+        # Store build cache per workflow.
+        disk-cache: ${{ github.workflow }}-${{ hashFiles('.github/workflows/wheels.yml') }}
+        # Share repository cache between workflows.
+        repository-cache: true
+
+    - name: Verify bazel installation
+      run: |
+        which bazel
+        bazel info
+        bazel version
+
+    - name: Install build
+      run: python -m pip install --upgrade pip build
+
+    - name: Build wheels
+      run: |
+        package_build/initialize.sh
+        python -m build --wheel package_build/tfx/
+        python -m build --wheel package_build/ml-pipelines-sdk/
+        mkdir wheelhouse
+        mv dist/*.whl wheelhouse/
+
+    - name: List and check wheels
+      run: |
+        pip install twine pkginfo>=1.10.0
+        ${{ matrix.ls || 'ls -lh' }} wheelhouse/
+        twine check wheelhouse/*
+
+    - name: Upload wheels
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheels-${{ matrix.python-version }}-${{ matrix.os }}
+        path: ./wheelhouse/*.whl
+
+  upload_to_pypi:
+    name: Upload to PyPI
+    runs-on: ubuntu-latest
+    if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch')
+    needs: [build_wheels, build_sdist]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/tfx
+    permissions:
+      id-token: write
+    steps:
+      - name: Retrieve wheels and sdist
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: wheels/
+
+      - name: List the build artifacts
+        run: |
+          ls -lAs wheels/
+
+      - name: Upload to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1.9
+        with:
+          packages_dir: wheels/

.gitignore

@@ -141,3 +141,6 @@ bazel-*
 **/*_pb2.py
 **/*_pb2_grpc.py
 # LINT.ThenChange(.dockerignore)
+
+MODULE.bazel
+MODULE.bazel.lock