From a2837efbf52faf5325a78ad54af8fc5efd2f23be Mon Sep 17 00:00:00 2001 From: Marco Zille Date: Fri, 10 Nov 2023 10:46:15 +0100 Subject: [PATCH] Added support for GitLab as provider Signed-off-by: Marco Zille --- .env.development | 2 +- database/controllers/repo.controller.js | 12 +- database/controllers/user.controller.js | 37 ++- database/models/repo.model.js | 7 +- database/models/user.model.js | 6 +- src/badges/bronzeBadge.js | 20 +- src/helpers/github.js | 64 +++--- src/helpers/gitlab.js | 289 ++++++++++++++++++++++++ src/routes/github.js | 31 +-- src/routes/gitlab.js | 107 +++++++++ src/routes/index.js | 40 +++- 11 files changed, 550 insertions(+), 65 deletions(-) create mode 100644 src/helpers/gitlab.js create mode 100644 src/routes/gitlab.js diff --git a/.env.development b/.env.development index 51f8b2f..671e80f 100644 --- a/.env.development +++ b/.env.development @@ -4,4 +4,4 @@ DB_PASSWORD=rootpwd DB_HOST=127.0.0.1 DB_DIALECT=mysql PORT=8000 -RETURN_JSON_ON_LOGIN=false \ No newline at end of file +RETURN_JSON_ON_LOGIN= \ No newline at end of file diff --git a/database/controllers/repo.controller.js b/database/controllers/repo.controller.js index a6f60e5..d04f335 100644 --- a/database/controllers/repo.controller.js +++ b/database/controllers/repo.controller.js @@ -3,23 +3,29 @@ const User = require("../models/user.model"); const saveRepo = async ( githubRepoId, + gitlabRepoId, DEICommitSHA, repoLink, badgeType, attachment, - name + userId ) => { + if ((githubRepoId && gitlabRepoId) || (!githubRepoId && !gitlabRepoId)) { + return "Error creating repo: provide either githubRepoId or gitlabRepoId"; + } + try { // Find a user by their name - const user = await User.findOne({ where: { name } }); + const user = await User.findOne({ where: { id: userId } }); if (!user) { - throw new Error(`User with name '${name}' not found.`); + throw new Error(`User with id '${userId}' not found.`); } // Create a new repo associated with the user const repo = await Repo.create({ githubRepoId, + gitlabRepoId, DEICommitSHA, repoLink, badgeType, diff --git a/database/controllers/user.controller.js b/database/controllers/user.controller.js index 8069e71..f840238 100644 --- a/database/controllers/user.controller.js +++ b/database/controllers/user.controller.js @@ -1,9 +1,23 @@ const User = require("../models/user.model"); -const saveUser = async (login, name, email, githubId) => { +const findUser = async (userId) => { + return await User.findOne({ where: { id: userId } }); +}; + +const saveUser = async (login, name, email, githubId, gitlabId) => { + if ((githubId && gitlabId) || (!githubId && !gitlabId)) { + console.error("Error creating user: provide either githubId or gitlabId"); + return null; + } + try { - // Find a user by their GitHub ID - let user = await User.findOne({ where: { githubId } }); + // Find a user by their GitHub ID or GitLab ID + let user = null; + if (githubId) { + user = await User.findOne({ where: { githubId } }); + } else if (gitlabId) { + user = await User.findOne({ where: { gitlabId } }); + } if (!user) { // If the user doesn't exist, create a new one @@ -12,9 +26,10 @@ const saveUser = async (login, name, email, githubId) => { name, email, githubId, + gitlabId, }); - console.log("New user created"); - return `New user created: ${user.login}`; + console.log(`New user created: ${user.login}`); + return user; } else { // User already exists; update if necessary const updates = []; @@ -34,15 +49,19 @@ const saveUser = async (login, name, email, githubId) => { if (updates.length > 0) { await user.save(); - return `User ${user.login} updated: ${updates.join(", ")}`; + console.log(`User ${user.login} updated: ${updates.join(", ")}`); } console.log("User Already Exists"); - return "User Already Exists"; + return user; } } catch (error) { - return `Error saving user: ${error.message}`; + console.error(`Error saving user: ${error.message}`); + return null; } }; -module.exports = saveUser; +module.exports = { + saveUser, + findUser, +}; diff --git a/database/models/repo.model.js b/database/models/repo.model.js index 54aec1e..e5dd14e 100644 --- a/database/models/repo.model.js +++ b/database/models/repo.model.js @@ -6,7 +6,11 @@ const User = require("./user.model"); const Repo = sequelize.define("repos", { githubRepoId: { type: DataTypes.INTEGER, - allowNull: false, + allowNull: true, + }, + gitlabRepoId: { + type: DataTypes.INTEGER, + allowNull: true, }, DEICommitSHA: { type: DataTypes.STRING, @@ -26,6 +30,7 @@ const Repo = sequelize.define("repos", { }, userId: { type: DataTypes.INTEGER, + allowNull: false, references: { model: "users", key: "id", diff --git a/database/models/user.model.js b/database/models/user.model.js index db025ee..446b040 100644 --- a/database/models/user.model.js +++ b/database/models/user.model.js @@ -16,7 +16,11 @@ const User = sequelize.define("users", { }, githubId: { type: DataTypes.INTEGER, - allowNull: false, + allowNull: true, + }, + gitlabId: { + type: DataTypes.INTEGER, + allowNull: true, }, }); diff --git a/src/badges/bronzeBadge.js b/src/badges/bronzeBadge.js index bf5521b..057ce4f 100644 --- a/src/badges/bronzeBadge.js +++ b/src/badges/bronzeBadge.js @@ -2,7 +2,16 @@ const augurAPI = require("../helpers/augurAPI"); const mailer = require("../helpers/mailer"); const saveRepo = require("../../database/controllers/repo.controller"); -const bronzeBadge = async (name, email, id, url, content, DEICommitSHA) => { +const bronzeBadge = async ( + userId, + userName, + email, + githubRepoId, + gitlabRepoId, + url, + content, + DEICommitSHA +) => { // Check for specific titles const titlesToCheck = [ "Project Access", @@ -22,7 +31,7 @@ const bronzeBadge = async (name, email, id, url, content, DEICommitSHA) => { } if (!hasAllTitles) { - mailer(email, name, "Bronze", null, null, results.join("\n")); + mailer(email, userName, "Bronze", null, null, results.join("\n")); } else if (hasAllTitles) { // email content const markdownLink = @@ -31,16 +40,17 @@ const bronzeBadge = async (name, email, id, url, content, DEICommitSHA) => { "<img src="https://raw.githubusercontent.com/AllInOpenSource/BadgingAPI/main/src/assets/badges/bronze-badge.svg" alt="DEI Badging Bronze Badge" />"; // send email - mailer(email, name, "Bronze", markdownLink, htmlLink); + mailer(email, userName, "Bronze", markdownLink, htmlLink); // save repo to database and return repo id const repo_id = await saveRepo( - id, + githubRepoId, + gitlabRepoId, DEICommitSHA, url, "Bronze", markdownLink, - name + userId ); // use repo id in augur api call diff --git a/src/helpers/github.js b/src/helpers/github.js index 58a9925..eace84b 100644 --- a/src/helpers/github.js +++ b/src/helpers/github.js @@ -114,7 +114,12 @@ const getUserRepositories = async (octokit) => { } return { - repositories: repos.map((repo) => repo.full_name), + repositories: repos.map((repo) => { + return { + id: repo.id, + fullName: repo.full_name, + }; + }), errors: [], }; } catch (error) { @@ -128,20 +133,22 @@ const getUserRepositories = async (octokit) => { /** * Get the id and url of the provided repository path * @param {*} octokit An Octokit instance - * @param {*} owner The (username) owner of the repository - * @param {*} repositoryPath The path to the repository, without the owner prefix + * @param {*} repositoryId The id of the repository * @returns A json object with `info` (the repository infos) and `errors` */ -const getRepositoryInfo = async (octokit, owner, repositoryPath) => { +const getRepositoryInfo = async (octokit, repositoryId) => { try { const { - data: { id, html_url }, - } = await octokit.repos.get({ owner, repo: repositoryPath }); + data: { id, html_url, full_name }, + } = await octokit.request("GET /repositories/{repositoryId}", { + repositoryId, + }); return { info: { id, url: html_url, + fullName: full_name, }, errors: [], }; @@ -156,23 +163,17 @@ const getRepositoryInfo = async (octokit, owner, repositoryPath) => { /** * Get the content and commit SHA of a file inside a repository * @param {*} octokit An Octokit instance - * @param {*} owner The (username) owner of the repository - * @param {*} repositoryPath The path to the repository, without the owner prefix + * @param {*} repositoryFullName The full path to the repository * @param {*} filePath The path to the file inside the repository * @returns A json object with `file` (SHA and content) and `errors` */ -const getFileContentAndSHA = async ( - octokit, - owner, - repositoryPath, - filePath -) => { +const getFileContentAndSHA = async (octokit, repositoryFullName, filePath) => { try { const { data: { sha, content }, } = await octokit.repos.getContent({ - owner, - repo: repositoryPath, + owner: repositoryFullName.split("/")[0], + repo: repositoryFullName.split("/")[1], path: filePath, }); @@ -193,23 +194,20 @@ const getFileContentAndSHA = async ( /** * Scans a list of repositories to try and apply for a badge + * @param {*} userId Id of the user * @param {*} name Full name of the user * @param {*} email User email used to send them emails with the results - * @param {*} repositories List of repositories to scan + * @param {*} repositoryIds List of repositories id to scan */ -const scanRepositories = async (name, email, repositories) => { +const scanRepositories = async (userId, name, email, repositoryIds) => { const octokit = new Octokit(); let results = []; try { - for (const repository of repositories) { - const owner = repository.split("/")[0]; - const repositoryPath = repository.split("/")[1]; - + for (const repositoryId of repositoryIds) { const { info, errors: info_errors } = await getRepositoryInfo( octokit, - owner, - repositoryPath + repositoryId ); if (info_errors.length > 0) { console.error(info_errors); @@ -218,8 +216,7 @@ const scanRepositories = async (name, email, repositories) => { const { file, errors: file_errors } = await getFileContentAndSHA( octokit, - owner, - repositoryPath, + info.fullName, "DEI.md" ); if (file_errors.length > 0) { @@ -230,7 +227,7 @@ const scanRepositories = async (name, email, repositories) => { try { // Check if the repo was badged before const existingRepo = await Repo.findOne({ - where: { githubRepoId: info.id }, + where: { githubRepoId: info.id, DEICommitSHA: file.sha }, }); if (file.content) { @@ -238,9 +235,11 @@ const scanRepositories = async (name, email, repositories) => { // Compare the DEICommitSHA with the existing repo's DEICommitSHA if (existingRepo.DEICommitSHA !== file.sha) { bronzeBadge( + userId, name, email, info.id, + null, info.url, file.content, file.sha @@ -251,7 +250,16 @@ const scanRepositories = async (name, email, repositories) => { } } else { // Repo not badged before, badge it - bronzeBadge(name, email, info.id, info.url, file.content, file.sha); + bronzeBadge( + userId, + name, + email, + info.id, + null, + info.url, + file.content, + file.sha + ); } } } catch (error) { diff --git a/src/helpers/gitlab.js b/src/helpers/gitlab.js new file mode 100644 index 0000000..c79e304 --- /dev/null +++ b/src/helpers/gitlab.js @@ -0,0 +1,289 @@ +const axios = require("axios"); +const Repo = require("../../database/models/repo.model.js"); +const bronzeBadge = require("../badges/bronzeBadge.js"); +const mailer = require("../helpers/mailer.js"); + +/** + * Starts the authorization process with the GitLab OAuth system + * @param {*} res Response to send back to the caller + */ +const authorizeApplication = (res) => { + if (!process.env.GITLAB_APP_CLIENT_ID) { + res.status(500).send("GitLab provider is not configured"); + return; + } + + const scopes = ["read_api"]; + const url = `https://gitlab.com/oauth/authorize?client_id=${ + process.env.GITLAB_APP_CLIENT_ID + }&response_type=code&state=STATE&scope=${scopes.join("+")}&redirect_uri=${ + process.env.GITLAB_APP_REDIRECT_URI + }`; + + res.redirect(url); +}; + +/** + * Calls the GitLab API to get an access token from the OAuth code. + * @param {*} code Code returned by the GitLab OAuth authorization API + * @returns A json object with `access_token` and `errors` + */ +const requestAccessToken = async (code) => { + try { + const { + data: { access_token }, + } = await axios.post( + "https://gitlab.com/oauth/token", + { + client_id: process.env.GITLAB_APP_CLIENT_ID, + client_secret: process.env.GITLAB_APP_CLIENT_SECRET, + code, + grant_type: "authorization_code", + redirect_uri: process.env.GITLAB_APP_REDIRECT_URI, + }, + { + headers: { + Accept: "application/json", + }, + } + ); + + return { + access_token, + errors: [], + }; + } catch (error) { + return { + access_token: "", + errors: [error.message], + }; + } +}; + +/** + * Calls the GitLab API to get the user info. + * @param {*} access_token Token used to authorize the call to the GitLab API + * @returns A json object with `user_info` and `errors` + */ +const getUserInfo = async (access_token) => { + try { + // Authenticated user details + const { + data: { username: login, name, email, id }, + } = await axios.get("https://gitlab.com/api/v4/user", { + headers: { + Accept: "application/json", + Authorization: `Bearer ${access_token}`, + }, + }); + + return { + user_info: { + login, + name, + email, + id, + }, + errors: [], + }; + } catch (error) { + return { + user_info: null, + errors: [error.message], + }; + } +}; + +/** + * Calls the GitLab API to get the user public repositories. + * @param {*} access_token Token used to authorize the call to the GitLab API + * @returns A json object with `repositories` and `errors` + */ +const getUserRepositories = async (access_token) => { + try { + // Authenticated user details + const { data } = await axios.get( + "https://gitlab.com/api/v4/projects?owned=true&visibility=public", + { + headers: { + Accept: "application/json", + Authorization: `Bearer ${access_token}`, + }, + } + ); + + return { + repositories: data.map((repo) => { + return { + id: repo.id, + fullName: repo.name_with_namespace, + }; + }), + errors: [], + }; + } catch (error) { + return { + repositories: null, + errors: [error.message], + }; + } +}; + +/** + * Get the id and url of the provided repository path + * @param {*} repositoryId The id of the repository + * @returns A json object with `info` (the repository infos) and `errors` + */ +const getRepositoryInfo = async (repositoryId) => { + try { + const { data } = await axios.get( + `https://gitlab.com/api/v4/projects/${repositoryId}`, + { + headers: { + Accept: "application/json", + }, + } + ); + + return { + info: { + id: repositoryId, + url: data.web_url, + defaultBranch: data.default_branch, + }, + errors: [], + }; + } catch (error) { + return { + info: null, + errors: [error.message], + }; + } +}; + +/** + * Get the content and commit SHA of a file inside a repository + * @param {*} repositoryId The path to the repository, without the owner prefix + * @param {*} filePath The path to the file inside the repository + * @param {*} branch Name of the branch to use as source for the file + * @returns A json object with `file` (SHA and content) and `errors` + */ +const getFileContentAndSHA = async (repositoryId, filePath, branch) => { + try { + const { data } = await axios.get( + `https://gitlab.com/api/v4/projects/${repositoryId}/repository/files/${filePath}?ref=${branch}`, + { + headers: { + Accept: "application/json", + }, + } + ); + + return { + file: { + sha: data.last_commit_id, + content: Buffer.from(data.content, "base64").toString(), + }, + errors: [], + }; + } catch (error) { + return { + file: null, + errors: [error.message], + }; + } +}; + +/** + * Scans a list of repositories to try and apply for a badge + * @param {*} userId Id of the user + * @param {*} name Full name of the user + * @param {*} email User email used to send them emails with the results + * @param {*} repositoryIds List of repositories id to scan + */ +const scanRepositories = async (userId, name, email, repositoryIds) => { + let results = []; + + try { + for (const repositoryId of repositoryIds) { + const { info, errors: info_errors } = await getRepositoryInfo( + repositoryId + ); + if (info_errors.length > 0) { + console.error(info_errors); + continue; + } + + const { file, errors: file_errors } = await getFileContentAndSHA( + repositoryId, + "DEI.md", + info.defaultBranch + ); + if (file_errors.length > 0) { + results.push(`${info.url} does not have a DEI.md file`); + continue; + } + + try { + // Check if the repo was badged before + const existingRepo = await Repo.findOne({ + where: { gitlabRepoId: info.id, DEICommitSHA: file.sha }, + }); + + if (file.content) { + if (existingRepo) { + // Compare the DEICommitSHA with the existing repo's DEICommitSHA + if (existingRepo.DEICommitSHA !== file.sha) { + bronzeBadge( + userId, + name, + email, + null, + info.id, + info.url, + file.content, + file.sha + ); + } else { + // Handle case when DEI.md file is not changed + results.push(`${info.url} was already badged`); + } + } else { + // Repo not badged before, badge it + bronzeBadge( + userId, + name, + email, + null, + info.id, + info.url, + file.content, + file.sha + ); + } + } + } catch (error) { + console.error(error.message); + } + } + + // Send one single email for generic errors while processing repositories + // The `bronzeBadge` function will handle sending email for each project + // with wether success or error messages + if (results.length > 0) { + mailer(email, name, "Bronze", null, null, results.join("\n")); + } + } catch (error) { + console.error("Error: ", error.message); + } + + return results; +}; + +module.exports = { + authorizeApplication, + requestAccessToken, + getUserInfo, + getUserRepositories, + scanRepositories, +}; diff --git a/src/routes/github.js b/src/routes/github.js index 9380845..8c9e945 100644 --- a/src/routes/github.js +++ b/src/routes/github.js @@ -1,6 +1,6 @@ const { Octokit } = require("@octokit/rest"); -const saveUser = require("../../database/controllers/user.controller.js"); +const { saveUser } = require("../../database/controllers/user.controller.js"); const github_helper = require("../helpers/github.js"); const handleOAuthCallback = async (req, res) => { @@ -24,12 +24,17 @@ const handleOAuthCallback = async (req, res) => { } // Save user to database - await saveUser( + const savedUser = await saveUser( user_info.login, user_info.name, user_info.email, - user_info.id + user_info.id, + null ); + if (!savedUser) { + res.status(500).send("Error saving user info"); + return; + } // Public repos they maintain, administer, or own const { repositories, errors: repositories_errors } = @@ -44,9 +49,10 @@ const handleOAuthCallback = async (req, res) => { process.env.RETURN_JSON_ON_LOGIN ) { res.status(200).json({ - name: user_info.name, - username: user_info.login, - email: user_info.email, + userId: savedUser.id, + name: savedUser.name, + username: savedUser.login, + email: savedUser.email, repos: repositories, provider: "github", }); @@ -57,20 +63,19 @@ const handleOAuthCallback = async (req, res) => { Repo List -

Welcome ${user_info.name}

-

Username: ${user_info.login}

-

Email: ${user_info.email}

+

Welcome ${savedUser.name}

+

Username: ${savedUser.login}

+

Email: ${savedUser.email}

- - +

Select Repositories:

${repositories .map( (repo) => `
- - + +
` ) diff --git a/src/routes/gitlab.js b/src/routes/gitlab.js new file mode 100644 index 0000000..c88409b --- /dev/null +++ b/src/routes/gitlab.js @@ -0,0 +1,107 @@ +const { saveUser } = require("../../database/controllers/user.controller.js"); +const gitlab_helper = require("../helpers/gitlab.js"); + +const handleOAuthCallback = async (req, res) => { + const code = req.body.code ?? req.query.code; + + const { access_token, errors: access_token_errors } = + await gitlab_helper.requestAccessToken(code); + if (access_token_errors.length > 0) { + res.status(500).send(access_token_errors.join()); + return; + } + + // Authenticated user details + const { user_info, errors: user_info_errors } = + await gitlab_helper.getUserInfo(access_token); + if (user_info_errors.length > 0) { + res.status(500).send(user_info_errors.join()); + return; + } + + // Save user to database + const savedUser = await saveUser( + user_info.login, + user_info.name, + user_info.email, + null, + user_info.id + ); + if (!savedUser) { + res.status(500).send("Error saving user info"); + return; + } + + // Public repos they maintain, administer, or own + const { repositories, errors: repositories_errors } = + await gitlab_helper.getUserRepositories(access_token); + if (repositories_errors.length > 0) { + res.status(500).send(repositories_errors.join()); + return; + } + + if ( + process.env.NODE_ENV === "production" || + process.env.RETURN_JSON_ON_LOGIN + ) { + res.status(200).json({ + userId: savedUser.id, + name: savedUser.name, + username: savedUser.login, + email: savedUser.email, + repos: repositories, + provider: "gitlab", + }); + } else if (process.env.NODE_ENV === "development") { + res.status(200).send(` + + + Repo List + + +

Welcome ${savedUser.name}

+

Username: ${savedUser.login}

+

Email: ${savedUser.email}

+ + + +

Select Repositories:

+ ${repositories + .map( + (repo) => ` +
+ + +
+ ` + ) + .join("")} +
+ +
+ + + `); + } else { + res.status(500).send("Unknown process mode"); + } +}; + +/** + * Sets up the provided Express app routes for GitLab + * @param {*} app Express application instance + */ +const setupGitLabRoutes = (app) => { + if ( + process.env.NODE_ENV === "production" || + process.env.RETURN_JSON_ON_LOGIN + ) { + app.post("/api/callback/gitlab", handleOAuthCallback); + } else if (process.env.NODE_ENV === "development") { + app.get("/api/callback/gitlab", handleOAuthCallback); + } +}; + +module.exports = { + setupGitLabRoutes, +}; diff --git a/src/routes/index.js b/src/routes/index.js index 2dc7c77..e24f2be 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -1,6 +1,9 @@ +const { findUser } = require("../../database/controllers/user.controller.js"); const Repo = require("../../database/models/repo.model.js"); const github_helpers = require("../helpers/github.js"); const github_routes = require("./github.js"); +const gitlab_helpers = require("../helpers/gitlab.js"); +const gitlab_routes = require("./gitlab.js"); /** * Redirects the user to the GitHub OAuth login page for authentication. @@ -12,6 +15,8 @@ const login = (req, res) => { if (provider === "github") { github_helpers.authorizeApplication(res); + } else if (provider === "gitlab") { + gitlab_helpers.authorizeApplication(res); } else { res.status(400).send(`Unknown provider: ${provider}`); } @@ -19,19 +24,45 @@ const login = (req, res) => { const reposToBadge = async (req, res) => { const selectedRepos = (await req.body.repos) || []; - const name = req.body.name || ""; - const email = req.body.email || ""; + const userId = req.body.userId; const provider = req.body.provider; if (!provider) { res.status(400).send("provider missing"); + return; + } + + if (!userId) { + res.status(400).send("userId missing"); + return; + } + + let user = null; + try { + user = await findUser(userId); + if (!user) { + res.status(404).json("User not found"); + return; + } + } catch (error) { + res.status(500).json("Error fetching user data"); + return; } // Process the selected repos as needed if (provider === "github") { const results = await github_helpers.scanRepositories( - name, - email, + user.id, + user.name, + user.email, + selectedRepos + ); + res.status(200).json({ results }); + } else if (provider === "gitlab") { + const results = await gitlab_helpers.scanRepositories( + user.id, + user.name, + user.email, selectedRepos ); res.status(200).json({ results }); @@ -71,6 +102,7 @@ const setupRoutes = (app) => { app.post("/api/repos-to-badge", reposToBadge); github_routes.setupGitHubRoutes(app); + gitlab_routes.setupGitLabRoutes(app); }; module.exports = {