Feat: Have primary/fallback package publishing methods (#168) #85

Workflow file for this run

.github/workflows/repository.yaml at c6e7f2e

	---
	# SPDX-License-Identifier: Apache-2.0
	# SPDX-FileCopyrightText: 2024 The Linux Foundation <https://linuxfoundation.org>

	name: "🤖 Repository DevOps Automation"

	# yamllint disable-line rule:truthy
	on:
	workflow_dispatch:
	push:
	branches: [main, master]

	env:
	DEFAULT_PYTHON: "3.10"
	BUILD_ARTEFACTS: "dist"
	# Configures publishing to PyPI
	PYPI_PUBLISHING: "true"
	# Create GitHub releases for all builds, not just production builds
	GITHUB_RELEASE_ALWAYS: "true"
	# Create an initial tag, if missing
	CREATE_MISSING_TAG: "true"

	jobs:
	classify-content:
	name: "Inspect Repository"
	runs-on: ubuntu-latest
	outputs:
	python: ${{ steps.classify.outputs.python }}
	notebooks: ${{ steps.classify.outputs.notebooks }}

	steps:
	- uses: actions/checkout@v4

	- name: "Classify content"
	id: classify
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/repository-classify-content@main

	workflow-capture-metadata:
	name: "Gather Workflow Metadata"
	runs-on: ubuntu-latest
	outputs:
	owner: ${{ steps.set.outputs.owner }}
	repository: ${{ steps.set.outputs.repository }}
	tagged: ${{ steps.set.outputs.tagged }}

	steps:
	- name: "Capture workflow metadata"
	id: set
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/workflow-capture-metadata@main

	python-project:
	name: "Python Project"
	needs:
	- classify-content
	if: needs.classify-content.outputs.python == 'true'
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.python.outputs.matrixjson }}
	permissions:
	# IMPORTANT: mandatory to raise the PR
	id-token: write
	pull-requests: write
	repository-projects: write
	contents: write

	steps:
	- uses: actions/checkout@v4

	- name: "Extract Python versioning"
	id: python
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/python-versions-matrix@main

	python-build:
	name: "Python Build"
	needs:
	- workflow-capture-metadata
	- python-project
	runs-on: "ubuntu-latest"
	continue-on-error: false
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(needs.python-project.outputs.matrix) }}
	permissions:
	contents: write
	# Required by SigStore signing action
	id-token: write
	outputs:
	publish: ${{ steps.python-project-build.outputs.publish }}

	steps:
	- uses: actions/checkout@v4

	- name: "Set up Python ${{ matrix.python-version }}"
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Install PDM tooling"
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Action: semantic-tag-latest"
	id: semantic-tag-latest
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/semantic-tag-latest@main

	- name: "Create initial v0.0.1 tag [conditional]"
	id: set-initial-tag
	if: steps.semantic-tag-latest.outputs.tag-missing == 'true' && env.CREATE_MISSING_TAG
	uses: softprops/action-gh-release@v2
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	prerelease: true
	tag_name: v0.0.1

	- name: "Build: Python project"
	id: python-project-build
	if: steps.semantic-tag-latest.outputs.missing == 'false'
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/python-project-build@main
	with:
	artefact_output_path: "dist"
	purge_output_path: "true"

	- name: "Validate artefacts with Twine"
	id: python-twine-check
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/python-twine-check@main

	- name: "Upload build artefacts"
	uses: actions/upload-artifact@v4
	if: matrix.python-version == env.DEFAULT_PYTHON
	with:
	name: ${{ github.ref_name }}
	path: ${{ env.BUILD_ARTEFACTS }}

	- name: "Sign packages with SigStore"
	uses: sigstore/[email protected]
	if: matrix.python-version == env.DEFAULT_PYTHON
	env:
	package-path: ${{ env.BUILD_ARTEFACTS }}
	with:
	inputs: >-
	./${{ env.BUILD_ARTEFACTS }}/*.tar.gz
	./${{ env.BUILD_ARTEFACTS }}/*.whl

	github:
	name: "Publish to GitHub"
	# Only publish on tag pushes
	needs: python-build
	runs-on: ubuntu-latest
	permissions:
	# IMPORTANT: mandatory to publish artefacts
	contents: write
	# Ensure development builds are NOT uploaded when build naming is broken
	# if: github.ref_name != 'main'
	steps:
	- name: "⬇ Download build artefacts"
	uses: actions/download-artifact@v4
	with:
	name: ${{ github.ref_name }}
	path: ${{ env.BUILD_ARTEFACTS }}

	- name: "Publish DEVELOPMENT artefacts to GitHub"
	if: (startsWith(github.ref, 'refs/tags/') != true) && (env.GITHUB_RELEASE_ALWAYS == 'true')
	uses: softprops/action-gh-release@v2
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	prerelease: true
	tag_name: ${{ github.ref_name }}-dev
	name: "Test/Development Build: ${{ github.ref_name }}"
	# body_path: ${{ github.workspace }}/CHANGELOG.rst
	files: \|
	${{ env.BUILD_ARTEFACTS }}/*.tar.gz
	${{ env.BUILD_ARTEFACTS }}/*.whl
	${{ env.BUILD_ARTEFACTS }}/.sigstore

	- name: "Publish PRODUCTION artefacts to GitHub"
	if: startsWith(github.ref, 'refs/tags/')
	uses: softprops/action-gh-release@v2
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	prerelease: false
	tag_name: ${{ github.ref_name }}
	name: "Test/Development Build: ${{ github.ref_name }}"
	# body_path: ${{ github.workspace }}/CHANGELOG.rst
	files: \|
	${{ env.BUILD_ARTEFACTS }}/*.tar.gz
	${{ env.BUILD_ARTEFACTS }}/*.whl
	${{ env.BUILD_ARTEFACTS }}/.sigstore

	testpypi:
	name: "Test Package Publishing"
	# Only publish on tag pushes
	# if: startsWith(github.ref, 'refs/tags/')
	needs:
	- workflow-capture-metadata
	- python-build
	runs-on: ubuntu-latest
	environment:
	name: testpypi
	permissions:
	# IMPORTANT: mandatory for trusted publishing
	id-token: write
	steps:
	- name: "Download build artefacts"
	uses: actions/download-artifact@v4
	if: env.PYPI_PUBLISHING == 'true'
	with:
	name: ${{ github.ref_name }}
	path: ${{ env.BUILD_ARTEFACTS }}

	- name: "Manicure artefacts directory"
	id: files
	run: \|
	# Remove file types unsupported by the Python Package Index
	if [ ! -d ${{ env.BUILD_ARTEFACTS }} ]; then
	echo "Early exit; build artefacts path NOT found: ${{ env.BUILD_ARTEFACTS }}"
	exit 0
	fi
	if [ -f ${{ env.BUILD_ARTEFACTS }}/buildvars.txt ]; then
	rm ${{ env.BUILD_ARTEFACTS }}/buildvars.txt
	else
	echo "No buildvars.txt file to purge"
	fi
	# Remove outputs related to SigStore signing
	if test -n "$(find ${{ env.BUILD_ARTEFACTS }} -maxdepth 1 -name '*.sigstore' -print -quit)"
	then
	echo "Found SigStore signing artefacts to purge"
	rm ${{ env.BUILD_ARTEFACTS }}/.sigstore
	else
	echo "No SigStore signing artefacts to purge"
	fi

	- name: "Check presence in test PyPI"
	id: url-check
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/url-validity-check@main
	with:
	prefix: "https://test.pypi.org/project"
	# Use project name, e.g. "/ITR"
	string: "/${{ needs.workflow-capture-metadata.outputs.repository }}"
	suffix: "/"

	- name: "Test package publishing"
	uses: pypa/gh-action-pypi-publish@release/v1
	# Primary/default method uses trusted publishing
	if: steps.url-check.outputs.valid == 'true'
	with:
	repository-url: https://test.pypi.org/legacy/
	# Show checksum values
	print-hash: true
	packages-dir: ${{ env.BUILD_ARTEFACTS }}
	# We already validate earlier in the pipeline
	verify-metadata: false
	# Test releases are always debugged
	verbose: true

	- name: "Test package publishing"
	uses: pypa/gh-action-pypi-publish@release/v1
	# Fallback method uses static organisation credentials
	# Used initially when trusted publishing is unavailable
	if: steps.url-check.outputs.valid == 'false'
	with:
	repository-url: https://test.pypi.org/legacy/
	# Show checksum values
	print-hash: true
	packages-dir: ${{ env.BUILD_ARTEFACTS }}
	# We already validate earlier in the pipeline
	verify-metadata: false
	# Test releases are always debugged
	verbose: true
	# Organisation secret/variable
	# Defined/stored in 1Password
	user: osclimate
	password: ${{ secrets.OSC_PYPI_TEST }}
	pypi:
	name: "Publish Package"
	# Only publish on tag pushes
	if:
	startsWith(github.ref, 'refs/tags/') &&
	needs.python-build.outputs.publish == 'true'
	# contains(github.event.head_commit.message, '[release]')
	needs: [python-build, workflow-capture-metadata, testpypi]
	runs-on: ubuntu-latest
	environment:
	name: pypi
	permissions:
	# IMPORTANT: mandatory for trusted publishing
	id-token: write
	steps:
	- name: "Download build artefacts"
	uses: actions/download-artifact@v4
	with:
	name: ${{ github.ref_name }}
	path: ${{ env.BUILD_ARTEFACTS }}

	- name: "Remove unsupported artefacts/files"
	run: \|
	# Remove unsupported artefacts/files
	if (ls ${{ env.BUILD_ARTEFACTS }}/.sigstore); then
	rm ${{ env.BUILD_ARTEFACTS }}/.sigstore
	fi

	- name: "Check presence in PyPI"
	id: url-check
	# yamllint disable-line rule:line-length
	uses: os-climate/devops-reusable-workflows/.github/actions/url-validity-check@main
	with:
	prefix: "https://pypi.org/project"
	# Use project name, e.g. "/ITR"
	string: "/${{ needs.workflow-capture-metadata.outputs.repository }}"
	suffix: "/"

	- name: "Publish to PyPI"
	uses: pypa/gh-action-pypi-publish@release/v1
	# Primary/default method uses trusted publishing
	if: steps.url-check.outputs.valid == 'true'
	with:
	# Show checksum values
	print-hash: true
	packages-dir: ${{ env.BUILD_ARTEFACTS }}
	# We already validate earlier in the pipeline
	verify-metadata: false

	- name: "Publish to PyPI"
	uses: pypa/gh-action-pypi-publish@release/v1
	# Fallback method uses static organisation credentials
	# Used initially when trusted publishing is unavailable
	if: steps.url-check.outputs.valid == 'false'
	with:
	# Show checksum values
	print-hash: true
	packages-dir: ${{ env.BUILD_ARTEFACTS }}
	# We already validate earlier in the pipeline
	verify-metadata: false
	# Organisation secret/variable
	# Defined/stored in 1Password
	user: osclimate
	password: ${{ secrets.OSC_PYPI_PROD }}

	notebooks:
	name: "Jupyter Notebooks"
	needs:
	- classify-content
	- python-project
	runs-on: "ubuntu-latest"
	continue-on-error: false
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(needs.python-project.outputs.matrix) }}
	# Don't run when pull request is merged, only if Jupyter Notebooks are present
	if: needs.classify-content.outputs.notebooks == 'true'

	steps:
	- uses: actions/checkout@v4

	- name: "Setup Python"
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Set up Python ${{ matrix.python-version }}"
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Install PDM tooling"
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Install package dependencies"
	run: \|
	# Install build dependencies
	python -m pip install -q --upgrade pip
	pdm export -o requirements.txt
	if [ -f requirements.txt ]; then
	pip install -r requirements.txt
	fi
	pip install -q .
	pip install -q pytest nbmake

	- name: "Testing Jupyter notebooks"
	run: \|
	# Testing Jupyter notebooks
	echo "Installing required dependencies"
	pip install --upgrade -q pytest nbmake
	# Consider enabling the line below when debugging/testing
	# find . -name '*.ipynb'
	echo "Running command: pytest --nbmake -- */.ipynb"
	pytest --nbmake src//.ipynb --cov=src/devops_reusable_workflows
	# Might need an __init__.py file in tests folder for notebooks there to be tested?
	# https://stackoverflow.com/questions/47287721/coverage-py-warning-no-data-was-collected-no-data-collected
	# pytest --nbmake tests/test_*.ipynb --cov=tests

	# TEMP DISABLED - NEED TO CHECK - WHERE ARE THESE LOGS GENERATED???
	# - name: "Upload Logs"
	# if: always()
	# uses: actions/upload-artifact@v4
	# with:
	# name: debug-logs
	# path: /tmp/*.log
	# retention-days: 14

	security:
	name: "Security Audit"
	needs:
	- classify-content
	- python-project
	if: needs.classify-content.outputs.python == 'true'
	runs-on: "ubuntu-latest"
	continue-on-error: true
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(needs.python-project.outputs.matrix) }}
	steps:
	- name: "CheckoutrRepository"
	uses: actions/checkout@v4

	- name: "Set up Python ${{ matrix.python-version }}"
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Install PDM tooling"
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: "Install dependencies"
	run: \|
	pip install -q --upgrade pip
	pdm lock
	pdm export -o requirements.txt
	python -m pip install -q -r requirements.txt
	python -m pip install -q .
	pip install --upgrade -q setuptools
	pdm list --graph

	- name: "Perform package auditing"
	uses: pypa/[email protected]

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Feat: Have primary/fallback package publishing methods (#168) #85

Workflow file

Feat: Have primary/fallback package publishing methods (#168) #85

Jobs

Run details

Workflow file for this run