diff --git a/repos/rust-lang/crates-io-auth-action.toml b/repos/rust-lang/crates-io-auth-action.toml index 5efa2f6fe..f0f8c9bde 100644 --- a/repos/rust-lang/crates-io-auth-action.toml +++ b/repos/rust-lang/crates-io-auth-action.toml @@ -1,7 +1,7 @@ org = "rust-lang" name = "crates-io-auth-action" description = "Get a crates.io temporary access token" -bots = [] +bots = ["renovate"] [access.teams] crates-io-infra-admins = "write" diff --git a/sync-team/src/github/api/mod.rs b/sync-team/src/github/api/mod.rs index debac26ca..cbbf9beb8 100644 --- a/sync-team/src/github/api/mod.rs +++ b/sync-team/src/github/api/mod.rs @@ -319,9 +319,23 @@ impl fmt::Display for RepoPermission { } } +#[derive(serde::Deserialize, Debug)] +pub(crate) struct OrgAppInstallation { + #[serde(rename = "id")] + pub(crate) installation_id: u64, + pub(crate) app_id: u64, +} + +#[derive(serde::Deserialize, Debug)] +pub(crate) struct RepoAppInstallation { + pub(crate) name: String, +} + #[derive(serde::Deserialize, Debug, Clone)] pub(crate) struct Repo { pub(crate) node_id: String, + #[serde(rename = "id")] + pub(crate) repo_id: u64, pub(crate) name: String, #[serde(alias = "owner", deserialize_with = "repo_owner")] pub(crate) org: String, diff --git a/sync-team/src/github/api/read.rs b/sync-team/src/github/api/read.rs index bef860a53..b97fc945c 100644 --- a/sync-team/src/github/api/read.rs +++ b/sync-team/src/github/api/read.rs @@ -1,7 +1,8 @@ use crate::github::api::Ruleset; use crate::github::api::{ - BranchProtection, GraphNode, GraphNodes, GraphPageInfo, HttpClient, Login, Repo, RepoTeam, - RepoUser, Team, TeamMember, TeamRole, team_node_id, url::GitHubUrl, user_node_id, + BranchProtection, GraphNode, GraphNodes, GraphPageInfo, HttpClient, Login, OrgAppInstallation, + Repo, RepoAppInstallation, RepoTeam, RepoUser, Team, TeamMember, TeamRole, team_node_id, + url::GitHubUrl, user_node_id, }; use anyhow::Context as _; use reqwest::Method; @@ -20,6 +21,16 @@ pub(crate) trait GithubRead { /// Get the members of an org fn org_members(&self, org: &str) -> anyhow::Result>; + /// Get the app installations of an org + fn org_app_installations(&self, org: &str) -> anyhow::Result>; + + /// Get the repositories enabled for an app installation. + fn app_installation_repos( + &self, + installation_id: u64, + org: &str, + ) -> anyhow::Result>; + /// Get all teams associated with a org /// /// Returns a list of tuples of team name and slug @@ -160,6 +171,50 @@ impl GithubRead for GitHubApiRead { Ok(members) } + fn org_app_installations(&self, org: &str) -> anyhow::Result> { + #[derive(serde::Deserialize, Debug)] + struct InstallationPage { + installations: Vec, + } + + let mut installations = Vec::new(); + self.client.rest_paginated( + &Method::GET, + &GitHubUrl::orgs(org, "installations")?, + |response: InstallationPage| { + installations.extend(response.installations); + Ok(()) + }, + )?; + Ok(installations) + } + + fn app_installation_repos( + &self, + installation_id: u64, + org: &str, + ) -> anyhow::Result> { + #[derive(serde::Deserialize, Debug)] + struct InstallationPage { + repositories: Vec, + } + + let mut installations = Vec::new(); + + let url = format!("user/installations/{installation_id}/repositories"); + self.client + .rest_paginated( + &Method::GET, + &GitHubUrl::new(&url, org), + |response: InstallationPage| { + installations.extend(response.repositories); + Ok(()) + }, + ) + .with_context(|| format!("failed to send rest paginated request to {url}"))?; + Ok(installations) + } + fn org_teams(&self, org: &str) -> anyhow::Result> { let mut teams = Vec::new(); @@ -294,6 +349,7 @@ impl GithubRead for GitHubApiRead { query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { id + databaseId autoMergeAllowed description homepageUrl @@ -313,6 +369,7 @@ impl GithubRead for GitHubApiRead { // Equivalent of `node_id` of the Rest API id: String, // Equivalent of `id` of the Rest API + database_id: u64, auto_merge_allowed: Option, description: Option, homepage_url: Option, @@ -332,6 +389,7 @@ impl GithubRead for GitHubApiRead { .with_context(|| format!("failed to retrieve repo `{org}/{repo}`"))?; let repo = result.and_then(|r| r.repository).map(|repo_response| Repo { + repo_id: repo_response.database_id, node_id: repo_response.id, name: repo.to_string(), description: repo_response.description.unwrap_or_default(), diff --git a/sync-team/src/github/api/write.rs b/sync-team/src/github/api/write.rs index f24f6d741..45f759afa 100644 --- a/sync-team/src/github/api/write.rs +++ b/sync-team/src/github/api/write.rs @@ -245,6 +245,7 @@ impl GitHubWrite { if self.dry_run { Ok(Repo { node_id: String::from("ID"), + repo_id: 0, name: name.to_string(), org: org.to_string(), description: settings.description.clone(), @@ -287,6 +288,54 @@ impl GitHubWrite { Ok(()) } + pub(crate) fn add_repo_to_app_installation( + &self, + installation_id: u64, + repository_id: u64, + org: &str, + ) -> anyhow::Result<()> { + debug!("Adding repository {repository_id} to installation {installation_id}"); + if !self.dry_run { + self.client + .req( + Method::PUT, + &GitHubUrl::new( + &format!( + "user/installations/{installation_id}/repositories/{repository_id}" + ), + org, + ), + )? + .send()? + .custom_error_for_status()?; + } + Ok(()) + } + + pub(crate) fn remove_repo_from_app_installation( + &self, + installation_id: u64, + repository_id: u64, + org: &str, + ) -> anyhow::Result<()> { + debug!("Removing repository {repository_id} from installation {installation_id}"); + if !self.dry_run { + self.client + .req( + Method::DELETE, + &GitHubUrl::new( + &format!( + "user/installations/{installation_id}/repositories/{repository_id}" + ), + org, + ), + )? + .send()? + .custom_error_for_status()?; + } + Ok(()) + } + /// Update a team's permissions to a repo pub(crate) fn update_team_repo_permissions( &self, diff --git a/sync-team/src/github/mod.rs b/sync-team/src/github/mod.rs index b3bf215bf..8df78633b 100644 --- a/sync-team/src/github/mod.rs +++ b/sync-team/src/github/mod.rs @@ -8,7 +8,7 @@ use crate::github::api::{GithubRead, Login, PushAllowanceActor, RepoPermission, use log::debug; use rust_team_data::v1::{Bot, BranchProtectionMode, MergeBot}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::fmt::Write; +use std::fmt::{Display, Formatter, Write}; pub(crate) use self::api::{GitHubApiRead, GitHubWrite, HttpClient}; @@ -30,6 +30,47 @@ pub(crate) fn create_diff( } type OrgName = String; +type RepoName = String; + +#[derive(Copy, Clone, Debug, PartialEq)] +enum GithubApp { + RenovateBot, + Bors, +} + +impl GithubApp { + /// You can find the GitHub app ID e.g. through `gh api apps/` or through the + /// app settings page (if we own the app). + fn from_id(app_id: u64) -> Option { + match app_id { + 2740 => Some(GithubApp::RenovateBot), + 278306 => Some(GithubApp::Bors), + _ => None, + } + } +} + +impl Display for GithubApp { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + GithubApp::RenovateBot => f.write_str("RenovateBot"), + GithubApp::Bors => f.write_str("Bors"), + } + } +} + +#[derive(Clone, Debug)] +struct OrgAppInstallation { + app: GithubApp, + installation_id: u64, + repositories: HashSet, +} + +#[derive(Clone, Debug, PartialEq)] +struct AppInstallation { + app: GithubApp, + installation_id: u64, +} struct SyncGitHub { github: Box, @@ -39,6 +80,7 @@ struct SyncGitHub { usernames_cache: HashMap, org_owners: HashMap>, org_members: HashMap>, + org_apps: HashMap>, } impl SyncGitHub { @@ -70,10 +112,29 @@ impl SyncGitHub { let mut org_owners = HashMap::new(); let mut org_members = HashMap::new(); + let mut org_apps = HashMap::new(); for org in &orgs { org_owners.insert((*org).to_string(), github.org_owners(org)?); org_members.insert((*org).to_string(), github.org_members(org)?); + + let mut installations: Vec = vec![]; + for installation in github.org_app_installations(org)? { + if let Some(app) = GithubApp::from_id(installation.app_id) { + let mut repositories = HashSet::new(); + for repo_installation in + github.app_installation_repos(installation.installation_id, org)? + { + repositories.insert(repo_installation.name); + } + installations.push(OrgAppInstallation { + app, + installation_id: installation.installation_id, + repositories, + }); + } + } + org_apps.insert(org.to_string(), installations); } Ok(SyncGitHub { @@ -84,6 +145,7 @@ impl SyncGitHub { usernames_cache, org_owners, org_members, + org_apps, }) } @@ -394,6 +456,7 @@ impl SyncGitHub { .iter() .map(|(name, env)| (name.clone(), env.clone())) .collect(), + app_installations: self.diff_app_installations(expected_repo, &[])?, })); } }; @@ -422,15 +485,41 @@ impl SyncGitHub { auto_merge_enabled: expected_repo.auto_merge_enabled, }; + let existing_installations = self + .org_apps + .get(&expected_repo.org) + .map(|installations| { + installations + .iter() + .filter_map(|installation| { + // Only load installations from apps that we know about, to avoid removing + // unknown installations. + if installation.repositories.contains(&actual_repo.name) { + Some(AppInstallation { + app: installation.app, + installation_id: installation.installation_id, + }) + } else { + None + } + }) + .collect::>() + }) + .unwrap_or_default(); + let app_installation_diffs = + self.diff_app_installations(expected_repo, &existing_installations)?; + Ok(RepoDiff::Update(UpdateRepoDiff { org: expected_repo.org.clone(), name: actual_repo.name, repo_node_id: actual_repo.node_id, + repo_id: actual_repo.repo_id, settings_diff: (old_settings, new_settings), permission_diffs, branch_protection_diffs, ruleset_diffs, environment_diffs, + app_installation_diffs, })) } @@ -678,6 +767,64 @@ impl SyncGitHub { Ok(ruleset_diffs) } + fn diff_app_installations( + &self, + expected_repo: &rust_team_data::v1::Repo, + existing_installations: &[AppInstallation], + ) -> anyhow::Result> { + let mut diff = vec![]; + let mut found_apps = Vec::new(); + + // Find apps that should be enabled on the repository + for app in expected_repo.bots.iter().filter_map(|bot| match bot { + Bot::Renovate => Some(GithubApp::RenovateBot), + Bot::Bors => Some(GithubApp::Bors), + Bot::Highfive + | Bot::Rfcbot + | Bot::RustTimer + | Bot::Rustbot + | Bot::Craterbot + | Bot::Glacierbot + | Bot::LogAnalyzer + | Bot::HerokuDeployAccess => None, + }) { + // Find installation ID of this app on GitHub + let gh_installation = self + .org_apps + .get(&expected_repo.org) + .and_then(|installations| { + installations + .iter() + .find(|installation| installation.app == app) + .map(|i| i.installation_id) + }); + let Some(gh_installation) = gh_installation else { + log::warn!( + "Application {app} should be enabled for repository {}/{}, but it is not installed on GitHub", + expected_repo.org, + expected_repo.name + ); + continue; + }; + let installation = AppInstallation { + app, + installation_id: gh_installation, + }; + found_apps.push(installation.clone()); + + if !existing_installations.contains(&installation) { + diff.push(AppInstallationDiff::Add(installation)); + } + } + for existing in existing_installations { + if !found_apps.contains(existing) { + diff.push(AppInstallationDiff::Remove(existing.clone())); + } + } + + Ok(diff) + } + fn expected_role(&self, org: &str, user: u64) -> TeamRole { if let Some(true) = self .org_owners @@ -1120,6 +1267,7 @@ struct CreateRepoDiff { branch_protections: Vec<(String, api::BranchProtection)>, rulesets: Vec, environments: Vec<(String, rust_team_data::v1::Environment)>, + app_installations: Vec, } impl CreateRepoDiff { @@ -1152,6 +1300,10 @@ impl CreateRepoDiff { sync.create_environment(&self.org, &self.name, env_name, &env.branches, &env.tags)?; } + for installation in &self.app_installations { + installation.apply(sync, repo.repo_id, &self.org)?; + } + Ok(()) } } @@ -1166,6 +1318,7 @@ impl std::fmt::Display for CreateRepoDiff { branch_protections, rulesets, environments, + app_installations, } = self; let RepoSettings { @@ -1226,6 +1379,12 @@ impl std::fmt::Display for CreateRepoDiff { } } } + + writeln!(f, " App Installations:")?; + for diff in app_installations { + write!(f, "{diff}")?; + } + Ok(()) } } @@ -1235,12 +1394,14 @@ struct UpdateRepoDiff { org: String, name: String, repo_node_id: String, + repo_id: u64, // old, new settings_diff: (RepoSettings, RepoSettings), permission_diffs: Vec, branch_protection_diffs: Vec, ruleset_diffs: Vec, environment_diffs: Vec, + app_installation_diffs: Vec, } #[derive(Debug)] @@ -1268,11 +1429,13 @@ impl UpdateRepoDiff { org: _, name: _, repo_node_id: _, + repo_id: _, settings_diff, permission_diffs, branch_protection_diffs, ruleset_diffs, environment_diffs, + app_installation_diffs, } = self; settings_diff.0 == settings_diff.1 @@ -1280,6 +1443,7 @@ impl UpdateRepoDiff { && branch_protection_diffs.is_empty() && ruleset_diffs.is_empty() && environment_diffs.is_empty() + && app_installation_diffs.is_empty() } fn can_be_modified(&self) -> bool { @@ -1342,6 +1506,10 @@ impl UpdateRepoDiff { sync.edit_repo(&self.org, &self.name, &self.settings_diff.1)?; } + for app_installation in &self.app_installation_diffs { + app_installation.apply(sync, self.repo_id, &self.org)?; + } + Ok(()) } } @@ -1356,11 +1524,13 @@ impl std::fmt::Display for UpdateRepoDiff { org, name, repo_node_id: _, + repo_id: _, settings_diff, permission_diffs, branch_protection_diffs, ruleset_diffs, environment_diffs, + app_installation_diffs, } = self; writeln!(f, "📝 Editing repo '{org}/{name}':")?; @@ -1466,6 +1636,14 @@ impl std::fmt::Display for UpdateRepoDiff { } } + if !app_installation_diffs.is_empty() { + writeln!(f, " App installation changes:")?; + + for diff in app_installation_diffs { + write!(f, "{diff}")?; + } + } + Ok(()) } } @@ -2125,3 +2303,36 @@ impl std::fmt::Display for DeleteTeamDiff { Ok(()) } } + +#[derive(Debug)] +enum AppInstallationDiff { + Add(AppInstallation), + Remove(AppInstallation), +} + +impl AppInstallationDiff { + fn apply(&self, sync: &GitHubWrite, repo_id: u64, org: &str) -> anyhow::Result<()> { + match self { + AppInstallationDiff::Add(app) => { + sync.add_repo_to_app_installation(app.installation_id, repo_id, org)?; + } + AppInstallationDiff::Remove(app) => { + sync.remove_repo_from_app_installation(app.installation_id, repo_id, org)?; + } + } + Ok(()) + } +} + +impl std::fmt::Display for AppInstallationDiff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AppInstallationDiff::Add(app) => { + writeln!(f, " Install app {}", app.app) + } + AppInstallationDiff::Remove(app) => { + writeln!(f, " Remove app {}", app.app) + } + } + } +} diff --git a/sync-team/src/github/tests/mod.rs b/sync-team/src/github/tests/mod.rs index 429e1602d..58c2448cf 100644 --- a/sync-team/src/github/tests/mod.rs +++ b/sync-team/src/github/tests/mod.rs @@ -216,6 +216,7 @@ fn repo_change_description() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "foo", @@ -234,6 +235,7 @@ fn repo_change_description() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -255,6 +257,7 @@ fn repo_change_homepage() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -277,6 +280,7 @@ fn repo_change_homepage() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -346,6 +350,7 @@ fn repo_create() { ], rulesets: [], environments: [], + app_installations: [], }, ), ] @@ -374,6 +379,7 @@ fn repo_add_member() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -401,6 +407,7 @@ fn repo_add_member() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -428,6 +435,7 @@ fn repo_change_member_permissions() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -456,6 +464,7 @@ fn repo_change_member_permissions() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -478,6 +487,7 @@ fn repo_remove_member() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -505,6 +515,7 @@ fn repo_remove_member() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -529,6 +540,7 @@ fn repo_add_team() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -556,6 +568,7 @@ fn repo_add_team() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -578,6 +591,7 @@ fn repo_change_team_permissions() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -606,6 +620,7 @@ fn repo_change_team_permissions() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -628,6 +643,7 @@ fn repo_remove_team() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -655,6 +671,7 @@ fn repo_remove_team() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -677,6 +694,7 @@ fn repo_archive_repo() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -695,6 +713,7 @@ fn repo_archive_repo() { branch_protection_diffs: [], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -720,6 +739,7 @@ fn repo_add_branch_protection() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -770,6 +790,7 @@ fn repo_add_branch_protection() { ], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -813,6 +834,7 @@ fn repo_update_branch_protection() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -861,6 +883,7 @@ fn repo_update_branch_protection() { ], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -890,6 +913,7 @@ fn repo_remove_branch_protection() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -915,6 +939,7 @@ fn repo_remove_branch_protection() { ], ruleset_diffs: [], environment_diffs: [], + app_installation_diffs: [], }, ), ] @@ -986,6 +1011,7 @@ fn repo_environment_create() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -1019,6 +1045,7 @@ fn repo_environment_create() { }, ), ], + app_installation_diffs: [], }, ), ] @@ -1045,6 +1072,7 @@ fn repo_environment_delete() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -1070,6 +1098,7 @@ fn repo_environment_delete() { "staging", ), ], + app_installation_diffs: [], }, ), ] @@ -1111,6 +1140,7 @@ fn repo_environment_update() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -1140,6 +1170,7 @@ fn repo_environment_update() { "staging", ), ], + app_installation_diffs: [], }, ), ] @@ -1171,6 +1202,7 @@ fn repo_environment_update_branches() { org: "rust-lang", name: "repo1", repo_node_id: "0", + repo_id: 0, settings_diff: ( RepoSettings { description: "", @@ -1206,6 +1238,7 @@ fn repo_environment_update_branches() { new_tags: [], }, ], + app_installation_diffs: [], }, ), ] diff --git a/sync-team/src/github/tests/test_utils.rs b/sync-team/src/github/tests/test_utils.rs index 36cdb83e7..7a00ca389 100644 --- a/sync-team/src/github/tests/test_utils.rs +++ b/sync-team/src/github/tests/test_utils.rs @@ -10,7 +10,8 @@ use rust_team_data::v1::{ use crate::Config; use crate::github::api::{ - BranchProtection, GithubRead, Repo, RepoTeam, RepoUser, Team, TeamMember, TeamPrivacy, TeamRole, + BranchProtection, GithubRead, OrgAppInstallation, Repo, RepoAppInstallation, RepoTeam, + RepoUser, Team, TeamMember, TeamPrivacy, TeamRole, }; use crate::github::{ OrgMembershipDiff, RepoDiff, SyncGitHub, TeamDiff, api, construct_branch_protection, @@ -135,6 +136,7 @@ impl DataModel { repo.name.clone(), Repo { node_id: org.repos.len().to_string(), + repo_id: org.repos.len() as u64, name: repo.name.clone(), org: repo.org.clone(), description: repo.description.clone(), @@ -533,6 +535,18 @@ impl GithubRead for GithubMock { Ok(self.get_org(org).members.iter().cloned().collect()) } + fn org_app_installations(&self, _org: &str) -> anyhow::Result> { + Ok(vec![]) + } + + fn app_installation_repos( + &self, + _installation_id: u64, + _org: &str, + ) -> anyhow::Result> { + Ok(vec![]) + } + fn org_teams(&self, org: &str) -> anyhow::Result> { Ok(self .get_org(org)