Feat: Add URL check for project in PyPI (#99) #18
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
# SPDX-License-Identifier: Apache-2.0 | |
# SPDX-FileCopyrightText: 2024 The Linux Foundation <https://linuxfoundation.org> | |
name: "🤖 Repository DevOps Automation" | |
# yamllint disable-line rule:truthy | |
on: | |
workflow_dispatch: | |
push: | |
branches: [main, master] | |
# paths: [".github/actions/**", ".github/workflows/**"] | |
env: | |
DEFAULT-PYTHON: "3.10" | |
ARTEFACTS: "dist" | |
# Configures publishing to PyPI | |
PUBLISH: "true" | |
jobs: | |
### Test Individual Composite Actions ### | |
test: | |
name: "Validate Individual Actions" | |
runs-on: ubuntu-latest | |
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' | |
continue-on-error: true | |
steps: | |
- name: "🧳 Checkout Repository" | |
uses: actions/checkout@v4 | |
with: | |
# Does not currently work: https://github.com/actions/checkout/issues/1471 | |
fetch-tags: true | |
# latest-semantic-tag currently contains a workaround for this behaviour | |
- name: "Action: python-twine-check" | |
id: python-twine-check | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-twine-check@main | |
with: | |
path: "tests/twine" | |
- name: "Action: semantic-tag-latest" | |
id: semantic-tag-latest | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/semantic-tag-latest@main | |
- name: "Action: semantic-tag-increment" | |
id: semantic-tag-increment | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/semantic-tag-increment@main | |
with: | |
tag: "${{ steps.semantic-tag-latest.outputs.tag }}" | |
type: "minor" | |
### Primary Python Workflow Testing ### | |
identify-content: | |
name: "Identify Content" | |
runs-on: ubuntu-latest | |
outputs: | |
python: ${{ steps.identify-content.outputs.python }} | |
notebooks: ${{ steps.identify-content.outputs.notebooks }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Identify Content" | |
id: identify-content | |
run: | | |
# Inspect files in repository | |
# Check if repository contains a Python project | |
if [ -f pyproject.toml ]; then | |
echo "Python project metadata found" | |
echo "python=true" >> "$GITHUB_OUTPUT" | |
else | |
echo "python=false" >> "$GITHUB_OUTPUT" | |
fi | |
# Check if repository contains Jupyter Notebooks | |
NOTEBOOKS=$(find . -name '*.ipynb' | wc -l ) | |
if [ "$NOTEBOOKS" -ne 0 ]; then | |
echo "Jupyter notebooks found" | |
echo "notebooks=true" >> "$GITHUB_OUTPUT" | |
else | |
echo "notebooks=false" >> "$GITHUB_OUTPUT" | |
fi | |
python-project: | |
name: "Python Project" | |
needs: | |
- identify-content | |
if: needs.identify-content.outputs.python == 'true' | |
runs-on: ubuntu-latest | |
outputs: | |
matrix: ${{ steps.python.outputs.matrixjson }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Set up Python ${{ env.default-python }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ env.default-python }} | |
- name: "Setup PDM Build Tool" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ env.default-python }} | |
- name: "Update Python dependencies" | |
uses: pdm-project/update-deps-action@v1 | |
with: | |
sign-off-commit: "true" | |
token: ${{ secrets.GH_TOKEN }} | |
commit-message: "Chore: Update dependencies and pdm.lock [skip ci]" | |
pr-title: "Update Python module dependencies" | |
update-strategy: eager | |
# Whether to install PDM plugins before update | |
install-plugins: "false" | |
- name: "Export Dependencies" | |
id: export-dependencies | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-export-dependencies@main | |
- name: "Extract Python Versioning" | |
id: python | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-versions-matrix@main | |
python-build: | |
name: "Python Build" | |
needs: | |
- python-project | |
runs-on: "ubuntu-latest" | |
continue-on-error: false | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
permissions: | |
contents: write | |
# Required by SigStore signing action | |
id-token: write | |
outputs: | |
publish: ${{ steps.python-project-build.outputs.publish }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Setup PDM Build Tool" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Action: semantic-tag-latest" | |
id: semantic-tag-latest | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/semantic-tag-latest@main | |
- name: "Create initial tag" | |
id: set-initial-tag | |
if: steps.semantic-tag-latest.outputs.tag-missing == 'true' | |
# https://github.com/softprops/action-gh-release | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: true | |
tag_name: v0.0.1 | |
- name: "Build: Python Project" | |
id: python-project-build | |
if: steps.semantic-tag-latest.outputs.missing == 'false' | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-project-build@main | |
- name: "Validate Artefacts with Twine" | |
id: python-twine-check | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-twine-check@main | |
- name: "Store the distribution packages" | |
uses: actions/upload-artifact@v4 | |
if: matrix.python-version == env.default-python | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Sign packages with Sigstore" | |
uses: sigstore/[email protected] | |
if: matrix.python-version == env.default-python | |
env: | |
package-path: ${{ env.artefacts }} | |
with: | |
inputs: >- | |
./${{ env.artefacts }}/*.tar.gz | |
./${{ env.artefacts }}/*.whl | |
github: | |
name: "Publish to GitHub" | |
# Only publish on tag pushes | |
needs: python-build | |
runs-on: ubuntu-latest | |
permissions: | |
# IMPORTANT: mandatory to publish artefacts | |
contents: write | |
# Ensure development builds are NOT uploaded when build naming is broken | |
if: github.ref_name != 'main' | |
steps: | |
- name: "⬇ Download build artefacts" | |
uses: actions/download-artifact@v4 | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Set Environment Variables" | |
id: setenv | |
run: | | |
# vernum="${{ env.default-python }}.$(date +'%Y%m%d%H%M')" | |
datetime="$(date +'%Y%m%d%H%M')" | |
echo "datetime=${datetime}" >> "$GITHUB_OUTPUT" | |
- name: "Publish DEVELOPMENT artefacts to GitHub" | |
if: startsWith(github.ref, 'refs/tags/') != true | |
# https://github.com/softprops/action-gh-release | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: true | |
tag_name: ${{ github.ref_name }}-dev | |
name: "Test/Development Build: ${{ github.ref_name }}" | |
# body_path: ${{ github.workspace }}/CHANGELOG.rst | |
files: | | |
${{ env.artefacts }}/*.tar.gz | |
${{ env.artefacts }}/*.whl | |
${{ env.artefacts }}/*.sigstore* | |
- name: "Publish PRODUCTION artefacts to GitHub" | |
if: startsWith(github.ref, 'refs/tags/') | |
# https://github.com/softprops/action-gh-release | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: false | |
tag_name: ${{ github.ref_name }} | |
name: "Test/Development Build: ${{ github.ref_name }}" | |
# body_path: ${{ github.workspace }}/CHANGELOG.rst | |
files: | | |
${{ env.artefacts }}/*.tar.gz | |
${{ env.artefacts }}/*.whl | |
${{ env.artefacts }}/*.sigstore* | |
testpypi: | |
name: "Test Package Publishing" | |
# Only publish on tag pushes | |
# if: startsWith(github.ref, 'refs/tags/') | |
needs: python-build | |
runs-on: ubuntu-latest | |
environment: | |
name: testpypi | |
permissions: | |
# IMPORTANT: mandatory for trusted publishing | |
id-token: write | |
steps: | |
- name: "Check Test PyPI for Project" | |
id: url-check | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/url-validity-check@main | |
with: | |
prefix: "https://https://test.pypi.org/project" | |
string: "/ITR" | |
suffix: "/" | |
- name: "Download build artefacts" | |
uses: actions/download-artifact@v4 | |
if: env.publish == 'true' && steps.url-check.outputs.valid == 'true' | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Validate Build Artefacts" | |
id: files | |
run: | | |
if [ -f ${{ env.artefacts }}/buildvars.txt ]; then | |
rm ${{ env.artefacts }}/buildvars.txt | |
fi | |
if (ls ${{ env.artefacts }}/*.sigstore*); then | |
rm ${{ env.artefacts }}/*.sigstore* | |
fi | |
- name: "Test Package Publishing" | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
repository-url: https://test.pypi.org/legacy/ | |
verbose: true | |
packages-dir: ${{ env.artefacts }} | |
pypi: | |
name: "Publish Package" | |
# Only publish on tag pushes | |
if: | |
startsWith(github.ref, 'refs/tags/') && | |
needs.python-build.outputs.publish == 'true' | |
# contains(github.event.head_commit.message, '[release]') | |
needs: [python-build, testpypi] | |
runs-on: ubuntu-latest | |
environment: | |
name: pypi | |
permissions: | |
# IMPORTANT: mandatory for trusted publishing | |
id-token: write | |
steps: | |
- name: "Download build artefacts" | |
uses: actions/download-artifact@v4 | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Remove files unsupported by PyPi" | |
run: | | |
if (ls ${{ env.artefacts }}/*.sigstore*); then | |
rm ${{ env.artefacts }}/*.sigstore* | |
fi | |
# - name: "Publish to PyPI" | |
# uses: pypa/gh-action-pypi-publish@release/v1 | |
# with: | |
# verbose: true | |
# packages-dir: ${{ env.artefacts }} | |
notebooks: | |
name: "Jupyter Notebooks" | |
needs: | |
- identify-content | |
- python-project | |
runs-on: "ubuntu-latest" | |
continue-on-error: false | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
# Don't run when pull request is merged, only if Jupyter Notebooks are present | |
if: needs.identify-content.outputs.notebooks == 'true' | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Setup Python" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install/Setup PDM" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install Dependencies" | |
run: | | |
python -m pip install --upgrade pip | |
pdm export -o requirements.txt | |
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
pip install . | |
pip install pytest nbmake | |
- name: "Testing Jupyter Notebooks" | |
run: | | |
echo "Testing notebooks with:" | |
echo " pytest --nbmake -- **/*.ipynb" | |
ls | |
find . -name '*.ipynb' | |
pytest --nbmake src/*/*.ipynb --cov=src/devops_reusable_workflows | |
# Might need an __init__.py file in tests folder? | |
# https://stackoverflow.com/questions/47287721/coverage-py-warning-no-data-was-collected-no-data-collected | |
# pytest --nbmake tests/test_*.ipynb --cov=tests | |
- name: "Upload Logs" | |
if: always() | |
uses: actions/upload-artifact@v4 | |
with: | |
name: debug-logs | |
path: /tmp/*.log | |
retention-days: 14 | |
security: | |
name: "Security Audit" | |
needs: | |
- identify-content | |
- python-project | |
if: needs.identify-content.outputs.python == 'true' | |
runs-on: "ubuntu-latest" | |
continue-on-error: true | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
steps: | |
- name: "Checkout Repository" | |
uses: actions/checkout@v4 | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install/Setup PDM" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install Dependencies" | |
run: | | |
pip install --upgrade pip | |
pdm lock | |
pdm export -o requirements.txt | |
python -m pip install -r requirements.txt | |
python -m pip install . | |
pip install --upgrade setuptools | |
pdm list --graph | |
- name: "Security Audit" | |
uses: pypa/[email protected] |