Chore: Toggle actions/test to continue on error, perform all tests (#… #59
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
# SPDX-License-Identifier: Apache-2.0 | |
# SPDX-FileCopyrightText: 2024 The Linux Foundation <https://linuxfoundation.org> | |
name: "🤖 Repository DevOps Automation" | |
# yamllint disable-line rule:truthy | |
on: | |
workflow_dispatch: | |
push: | |
branches: [main, master] | |
env: | |
DEFAULT-PYTHON: "3.10" | |
ARTEFACTS: "dist" | |
# Configures publishing to PyPI | |
PYPI-PUBLISHING: "true" | |
GITHUB-RELEASE-DEV: "true" | |
jobs: | |
classify-content: | |
name: "Classify Repository" | |
runs-on: ubuntu-latest | |
outputs: | |
python: ${{ steps.classify.outputs.python }} | |
notebooks: ${{ steps.classify.outputs.notebooks }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Inspect Content" | |
id: classify | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/repository-classify-content@main | |
workflow-capture-metadata: | |
name: "Gather Workflow Metadata" | |
runs-on: ubuntu-latest | |
outputs: | |
owner: ${{ steps.set.outputs.owner }} | |
repository: ${{ steps.set.outputs.repository }} | |
tagged: ${{ steps.set.outputs.tagged }} | |
steps: | |
- name: "Capture workflow metadata" | |
id: set | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/workflow-capture-metadata@main | |
python-project: | |
name: "Python Project" | |
needs: | |
- classify-content | |
if: needs.classify-content.outputs.python == 'true' | |
runs-on: ubuntu-latest | |
outputs: | |
matrix: ${{ steps.python.outputs.matrixjson }} | |
permissions: | |
# IMPORTANT: mandatory to raise the PR | |
id-token: write | |
pull-requests: write | |
repository-projects: write | |
contents: write | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Extract Python versioning" | |
id: python | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-versions-matrix@main | |
python-build: | |
name: "Python Build" | |
needs: | |
- workflow-capture-metadata | |
- python-project | |
runs-on: "ubuntu-latest" | |
continue-on-error: false | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
permissions: | |
contents: write | |
# Required by SigStore signing action | |
id-token: write | |
outputs: | |
publish: ${{ steps.python-project-build.outputs.publish }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install PDM tooling" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Action: semantic-tag-latest" | |
id: semantic-tag-latest | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/semantic-tag-latest@main | |
- name: "Create initial tag" | |
id: set-initial-tag | |
if: steps.semantic-tag-latest.outputs.tag-missing == 'true' | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: true | |
tag_name: v0.0.1 | |
- name: "Build: Python project" | |
id: python-project-build | |
if: steps.semantic-tag-latest.outputs.missing == 'false' | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-project-build@main | |
- name: "Validate artefacts with Twine" | |
id: python-twine-check | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/python-twine-check@main | |
- name: "Store the distribution packages" | |
uses: actions/upload-artifact@v4 | |
if: matrix.python-version == env.default-python | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Sign packages with SigStore" | |
uses: sigstore/[email protected] | |
if: matrix.python-version == env.default-python | |
env: | |
package-path: ${{ env.artefacts }} | |
with: | |
inputs: >- | |
./${{ env.artefacts }}/*.tar.gz | |
./${{ env.artefacts }}/*.whl | |
github: | |
name: "Publish to GitHub" | |
# Only publish on tag pushes | |
needs: python-build | |
runs-on: ubuntu-latest | |
permissions: | |
# IMPORTANT: mandatory to publish artefacts | |
contents: write | |
# Ensure development builds are NOT uploaded when build naming is broken | |
# if: github.ref_name != 'main' | |
steps: | |
- name: "⬇ Download build artefacts" | |
uses: actions/download-artifact@v4 | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Publish DEVELOPMENT artefacts to GitHub" | |
if: (startsWith(github.ref, 'refs/tags/') != true) && (env.github-dev-release == 'true') | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: true | |
tag_name: ${{ github.ref_name }}-dev | |
name: "Test/Development Build: ${{ github.ref_name }}" | |
# body_path: ${{ github.workspace }}/CHANGELOG.rst | |
files: | | |
${{ env.artefacts }}/*.tar.gz | |
${{ env.artefacts }}/*.whl | |
${{ env.artefacts }}/*.sigstore* | |
- name: "Publish PRODUCTION artefacts to GitHub" | |
if: startsWith(github.ref, 'refs/tags/') | |
uses: softprops/action-gh-release@v2 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
prerelease: false | |
tag_name: ${{ github.ref_name }} | |
name: "Test/Development Build: ${{ github.ref_name }}" | |
# body_path: ${{ github.workspace }}/CHANGELOG.rst | |
files: | | |
${{ env.artefacts }}/*.tar.gz | |
${{ env.artefacts }}/*.whl | |
${{ env.artefacts }}/*.sigstore* | |
testpypi: | |
name: "Test Package Publishing" | |
# Only publish on tag pushes | |
# if: startsWith(github.ref, 'refs/tags/') | |
needs: | |
- workflow-capture-metadata | |
- python-build | |
runs-on: ubuntu-latest | |
environment: | |
name: testpypi | |
permissions: | |
# IMPORTANT: mandatory for trusted publishing | |
id-token: write | |
steps: | |
- name: "Check presence in test PyPI" | |
id: url-check | |
# yamllint disable-line rule:line-length | |
uses: os-climate/devops-reusable-workflows/.github/actions/url-validity-check@main | |
with: | |
prefix: "https://test.pypi.org/project" | |
# Use project name, e.g. "/ITR" | |
string: "/${{ needs.workflow-capture-metadata.outputs.repository }}" | |
suffix: "/" | |
- name: "Download build artefacts" | |
uses: actions/download-artifact@v4 | |
if: env.pypi-publishing == 'true' && steps.url-check.outputs.valid == 'true' | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Manicure artefacts directory" | |
id: files | |
run: | | |
# Remove file types unsupported by the Python Package Index | |
if [ ! -d ${{ env.artefacts }} ]; then | |
echo "Early exit; build artefacts path NOT found: ${{ env.artefacts }}" | |
exit 0 | |
fi | |
if [ -f ${{ env.artefacts }}/buildvars.txt ]; then | |
rm ${{ env.artefacts }}/buildvars.txt | |
else | |
echo "No buildvars.txt file to purge" | |
fi | |
# Remove outputs related to SigStore signing | |
if test -n "$(find ${{ env.artefacts }} -maxdepth 1 -name '**.sigstore*' -print -quit)" | |
then | |
echo "Found SigStore signing artefacts to purge" | |
rm ${{ env.artefacts }}/*.sigstore* | |
else | |
echo "No SigStore signing artefacts to purge" | |
fi | |
- name: "Test package publishing" | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
repository-url: https://test.pypi.org/legacy/ | |
verbose: true | |
packages-dir: ${{ env.artefacts }} | |
pypi: | |
name: "Publish Package" | |
# Only publish on tag pushes | |
if: | |
startsWith(github.ref, 'refs/tags/') && | |
needs.python-build.outputs.publish == 'true' | |
# contains(github.event.head_commit.message, '[release]') | |
needs: [python-build, testpypi] | |
runs-on: ubuntu-latest | |
environment: | |
name: pypi | |
permissions: | |
# IMPORTANT: mandatory for trusted publishing | |
id-token: write | |
steps: | |
- name: "Download build artefacts" | |
uses: actions/download-artifact@v4 | |
with: | |
name: ${{ github.ref_name }} | |
path: ${{ env.artefacts }} | |
- name: "Remove files unsupported by PyPi" | |
run: | | |
if (ls ${{ env.artefacts }}/*.sigstore*); then | |
rm ${{ env.artefacts }}/*.sigstore* | |
fi | |
- name: "Publish to PyPI" | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
verbose: true | |
packages-dir: ${{ env.artefacts }} | |
notebooks: | |
name: "Jupyter Notebooks" | |
needs: | |
- classify-content | |
- python-project | |
runs-on: "ubuntu-latest" | |
continue-on-error: false | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
# Don't run when pull request is merged, only if Jupyter Notebooks are present | |
if: needs.classify-content.outputs.notebooks == 'true' | |
steps: | |
- uses: actions/checkout@v4 | |
- name: "Setup Python" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install PDM tooling" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install package dependencies" | |
run: | | |
python -m pip install --upgrade pip | |
pdm export -o requirements.txt | |
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
pip install . | |
pip install pytest nbmake | |
- name: "Testing Jupyter Notebooks" | |
run: | | |
echo "Testing notebooks with:" | |
echo " pytest --nbmake -- **/*.ipynb" | |
ls | |
find . -name '*.ipynb' | |
pytest --nbmake src/*/*.ipynb --cov=src/devops_reusable_workflows | |
# Might need an __init__.py file in tests folder? | |
# https://stackoverflow.com/questions/47287721/coverage-py-warning-no-data-was-collected-no-data-collected | |
# pytest --nbmake tests/test_*.ipynb --cov=tests | |
# TEMP DISABLED - NEED TO CHECK - WHERE ARE THESE LOGS GENERATED??? | |
# - name: "Upload Logs" | |
# if: always() | |
# uses: actions/upload-artifact@v4 | |
# with: | |
# name: debug-logs | |
# path: /tmp/*.log | |
# retention-days: 14 | |
security: | |
name: "Security Audit" | |
needs: | |
- classify-content | |
- python-project | |
if: needs.classify-content.outputs.python == 'true' | |
runs-on: "ubuntu-latest" | |
continue-on-error: true | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJson(needs.python-project.outputs.matrix) }} | |
steps: | |
- name: "CheckoutrRepository" | |
uses: actions/checkout@v4 | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install PDM tooling" | |
uses: pdm-project/setup-pdm@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: "Install dependencies" | |
run: | | |
pip install --upgrade pip | |
pdm lock | |
pdm export -o requirements.txt | |
python -m pip install -r requirements.txt | |
python -m pip install . | |
pip install --upgrade setuptools | |
pdm list --graph | |
- name: "Perform package auditing" | |
uses: pypa/[email protected] |