From a3fbef6fc64457dbe67c19630a96d0fca506ec80 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 18:12:06 +0200 Subject: [PATCH 01/11] main: add TODO --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 05e8f20..b30da93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,8 @@ struct JobSender(Mutex>); #[post("/", data = "")] fn gitea_webhook(payload: Json, sender: State) -> Status { + // TODO: validate Gitea signature + { let sender = sender.0.lock().unwrap(); let repo = &payload.repository; From 7c21d65920b70e624932e20513bb2f1c99d5b933 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 20:11:37 +0200 Subject: [PATCH 02/11] job: log success when finished --- src/job.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index 25697b5..4142805 100644 --- a/src/job.rs +++ b/src/job.rs @@ -159,6 +159,10 @@ impl Job { self.update_repo()?; } - self.update_mirrors() + self.update_mirrors(config)?; + + info!("Done processing job {}!", self.repo.full_name); + + Ok(()) } } From 295a28bab4930e444d0b93b02bb0f7043aefb6ff Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 20:52:04 +0200 Subject: [PATCH 03/11] lohr: add default and additional remotes Co-authored-by: Antoine Martin --- Cargo.lock | 34 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/job.rs | 65 +++++++++++++++++++++++++++++++++++++------------ src/main.rs | 27 +++++++++++++++++--- src/settings.rs | 13 ++++++++++ 5 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 src/settings.rs diff --git a/Cargo.lock b/Cargo.lock index ce08ffb..8fcd247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,6 +231,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + [[package]] name = "filetime" version = "0.2.14" @@ -459,6 +465,12 @@ version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + [[package]] name = "log" version = "0.3.9" @@ -486,6 +498,7 @@ dependencies = [ "rocket", "rocket_contrib", "serde", + "serde_yaml", ] [[package]] @@ -851,6 +864,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + [[package]] name = "sha2" version = "0.9.3" @@ -1104,6 +1129,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yansi" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 14d5b4d..6aabd22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,4 @@ log = "0.4.14" rocket = "0.4.7" rocket_contrib = { version = "0.4.7", features = [ "json" ] } serde = { version = "1.0.125", features = [ "derive" ] } +serde_yaml = "0.8.17" diff --git a/src/job.rs b/src/job.rs index 4142805..fd2f094 100644 --- a/src/job.rs +++ b/src/job.rs @@ -8,6 +8,7 @@ use anyhow::bail; use log::info; use crate::gitea::Repository; +use crate::settings::{GlobalSettings, RepoUrl}; pub(crate) struct Job { repo: Repository, @@ -91,7 +92,9 @@ impl Job { Ok(()) } - fn get_remotes(&self) -> anyhow::Result { + /// Can return Ok(None) if the .lohr file didn't exist, but no significant error occured + fn read_remotes_from_lohr_file(&self) -> anyhow::Result>> { + // try to read .lohr file from bare repo (hence the git show sorcery) let output = Command::new("git") .arg("-C") .arg(format!("{}", self.local_path.as_ref().unwrap().display())) @@ -101,24 +104,56 @@ impl Job { if !output.status.success() { let error = str::from_utf8(&output.stderr)?; - let code = output - .status - .code() - .unwrap_or_else(|| output.status.signal().unwrap()); - bail!( - "couldn't read .lohr file from repo {}: exit code {}, stderr:\n{}", - self.repo.full_name, - code, - error - ); + // this error case is okay, .lohr just doesn't exist + if error.contains("does not exist in 'HEAD'") { + return Ok(None); + } else { + let code = output + .status + .code() + .unwrap_or_else(|| output.status.signal().unwrap()); + + bail!( + "couldn't read .lohr file from repo {}: exit code {}, stderr:\n{}", + self.repo.full_name, + code, + error + ); + } } - Ok(String::from_utf8(output.stdout)?) + let output = String::from_utf8(output.stdout)?; + + Ok(Some(output.lines().map(String::from).collect())) } - fn update_mirrors(&self) -> anyhow::Result<()> { - for remote in self.get_remotes()?.lines() { + fn get_remotes(&self, config: &GlobalSettings) -> anyhow::Result> { + let local_path = self.local_path.as_ref().unwrap(); + + let stem_to_repo = |stem: &RepoUrl| -> RepoUrl { + let mut res = stem.clone(); + if !res.ends_with('/') { + res.push('/'); + }; + res.push_str(local_path.file_name().unwrap().to_str().unwrap()); + res + }; + + // use either .lohr file or default remotes from config + let mut remotes = match self.read_remotes_from_lohr_file()? { + Some(remotes) if !remotes.is_empty() => remotes, + _ => config.default_remotes.iter().map(stem_to_repo).collect(), + }; + + // additional remotes + remotes.append(&mut config.additional_remotes.iter().map(stem_to_repo).collect()); + + Ok(remotes) + } + + fn update_mirrors(&self, config: &GlobalSettings) -> anyhow::Result<()> { + for remote in &self.get_remotes(config)? { info!("Updating mirror {}:{}...", remote, self.repo.full_name); let output = Command::new("git") @@ -148,7 +183,7 @@ impl Job { Ok(()) } - pub(crate) fn run(&mut self, homedir: &Path) -> anyhow::Result<()> { + pub(crate) fn run(&mut self, homedir: &Path, config: &GlobalSettings) -> anyhow::Result<()> { let local_path = homedir.join(&self.repo.full_name); assert!(local_path.is_absolute()); self.local_path = Some(local_path); diff --git a/src/main.rs b/src/main.rs index b30da93..6bb3613 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] use std::env; +use std::fs::File; use std::path::PathBuf; use std::sync::{ mpsc::{channel, Receiver, Sender}, @@ -19,6 +20,9 @@ use gitea::GiteaWebHook; mod job; use job::Job; +mod settings; +use settings::GlobalSettings; + struct JobSender(Mutex>); #[post("/", data = "")] @@ -34,29 +38,44 @@ fn gitea_webhook(payload: Json, sender: State) -> Statu Status::Ok } -fn repo_updater(rx: Receiver, homedir: PathBuf) { +fn repo_updater(rx: Receiver, homedir: PathBuf, config: GlobalSettings) { loop { let mut job = rx.recv().unwrap(); - if let Err(err) = job.run(&homedir) { + if let Err(err) = job.run(&homedir, &config) { error!("couldn't process job: {}", err); } } } -fn main() { +fn parse_config(mut path: PathBuf) -> anyhow::Result { + path.push("lohr-config"); + path.set_extension("yaml"); + let config = if let Ok(file) = File::open(path.as_path()) { + serde_yaml::from_reader(file)? + } else { + Default::default() + }; + Ok(config) +} + +fn main() -> anyhow::Result<()> { let (sender, receiver) = channel(); let homedir = env::var("LOHR_HOME").unwrap_or_else(|_| "./".to_string()); let homedir: PathBuf = homedir.into(); let homedir = homedir.canonicalize().expect("LOHR_HOME isn't valid!"); + let config = parse_config(homedir.clone())?; + thread::spawn(move || { - repo_updater(receiver, homedir); + repo_updater(receiver, homedir, config); }); rocket::ignite() .mount("/", routes![gitea_webhook]) .manage(JobSender(Mutex::new(sender))) .launch(); + + Ok(()) } diff --git a/src/settings.rs b/src/settings.rs new file mode 100644 index 0000000..bfe7744 --- /dev/null +++ b/src/settings.rs @@ -0,0 +1,13 @@ +use serde::Deserialize; + +pub(crate) type RepoUrl = String; // FIXME: probably needs a better type than this + +#[derive(Default, Deserialize)] +pub(crate) struct GlobalSettings { + /// List of remote stems to use when no `.lohr` file is found + #[serde(default)] + pub default_remotes: Vec, + /// List of remote stems to use for every repository + #[serde(default)] + pub additional_remotes: Vec, +} From 3ccf08f5bd56faddc4be5c588d19ecaa4431f0e2 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 20:54:19 +0200 Subject: [PATCH 04/11] job: remove redundant repo name from log --- src/job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index fd2f094..1a15233 100644 --- a/src/job.rs +++ b/src/job.rs @@ -154,7 +154,7 @@ impl Job { fn update_mirrors(&self, config: &GlobalSettings) -> anyhow::Result<()> { for remote in &self.get_remotes(config)? { - info!("Updating mirror {}:{}...", remote, self.repo.full_name); + info!("Updating mirror {}...", remote); let output = Command::new("git") .arg("-C") From 2954981e0ffbca7034016cde3e11b3846733ada1 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 18:00:01 +0000 Subject: [PATCH 05/11] nix: add 'flakes.nix' --- flake.lock | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 48 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..8f31c0b --- /dev/null +++ b/flake.lock @@ -0,0 +1,94 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1614513358, + "narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5466c5bbece17adaab2d82fae80b46e807611bf3", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "mozillapkgs": { + "flake": false, + "locked": { + "lastModified": 1603906276, + "narHash": "sha256-RsNPnEKd7BcogwkqhaV5kI/HuNC4flH/OQCC/4W5y/8=", + "owner": "mozilla", + "repo": "nixpkgs-mozilla", + "rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4", + "type": "github" + }, + "original": { + "owner": "mozilla", + "repo": "nixpkgs-mozilla", + "type": "github" + } + }, + "naersk": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1614785451, + "narHash": "sha256-TPw8kQvr2UNCuvndtY+EjyXp6Q5GEW2l9UafXXh1XmI=", + "owner": "nmattia", + "repo": "naersk", + "rev": "e0fe990b478a66178a58c69cf53daec0478ca6f9", + "type": "github" + }, + "original": { + "owner": "nmattia", + "ref": "master", + "repo": "naersk", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1617038346, + "narHash": "sha256-5AVL8TxnUT/8rX1XfxK05fP+jdf20jhg5gjZ2TLo/Wc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8e71416e5d4306c3800133ca6dc2ad4469480b90", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1616670887, + "narHash": "sha256-wn+l9qJfR5sj5Gq4DheJHAcBDfOs9K2p9seW2f35xzs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c0e881852006b132236cbf0301bd1939bb50867e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "mozillapkgs": "mozillapkgs", + "naersk": "naersk", + "nixpkgs": "nixpkgs_2" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..53fa44b --- /dev/null +++ b/flake.nix @@ -0,0 +1,48 @@ +{ + inputs = { + naersk.url = "github:nmattia/naersk/master"; + mozillapkgs = { + url = "github:mozilla/nixpkgs-mozilla"; + flake = false; + }; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, naersk, mozillapkgs, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + + mozilla = pkgs.callPackage (mozillapkgs + "/package-set.nix") { }; + rustNightly = (mozilla.rustChannelOf { + date = "2021-03-29"; + channel = "nightly"; + sha256 = "sha256-Y94CnslybZgiZlNVV6Cg0TUPV2OeDXakPev1kqdt9Kk="; + }).rust; + + naersk-lib = pkgs.callPackage naersk { + cargo = rustNightly; + rustc = rustNightly; + }; + in + { + defaultPackage = naersk-lib.buildPackage ./.; + + defaultApp = flake-utils.lib.mkApp { + drv = self.defaultPackage."${system}"; + }; + + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + nixpkgs-fmt + pre-commit + rustPackages.clippy + rustNightly + rustfmt + ]; + + RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc; + }; + }); +} From a961db0433aa70283a3b8d22c77d55783b8c9325 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 18:04:56 +0000 Subject: [PATCH 06/11] nix: support legacy nix using 'flake-compat' --- default.nix | 13 +++++++++++++ flake.lock | 17 +++++++++++++++++ flake.nix | 4 ++++ shell.nix | 13 +++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 default.nix create mode 100644 shell.nix diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..873ece4 --- /dev/null +++ b/default.nix @@ -0,0 +1,13 @@ +(import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + }).defaultNix diff --git a/flake.lock b/flake.lock index 8f31c0b..bcf4f30 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1606424373, + "narHash": "sha256-oq8d4//CJOrVj+EcOaSXvMebvuTkmBJuT5tzlfewUnQ=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "99f1c2157fba4bfe6211a321fd0ee43199025dbf", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1614513358, @@ -82,6 +98,7 @@ }, "root": { "inputs": { + "flake-compat": "flake-compat", "flake-utils": "flake-utils", "mozillapkgs": "mozillapkgs", "naersk": "naersk", diff --git a/flake.nix b/flake.nix index 53fa44b..c97297a 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,10 @@ }; nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; }; outputs = { self, naersk, mozillapkgs, nixpkgs, flake-utils, ... }: diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..9eb132a --- /dev/null +++ b/shell.nix @@ -0,0 +1,13 @@ +(import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + }).shellNix From edbe654df67828362b5450d810c9a28d6e55d5e4 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 18:06:44 +0000 Subject: [PATCH 07/11] nix: add 'direnv' integration --- .envrc | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..c23bb91 --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +use_flake() { + watch_file flake.nix + watch_file flake.lock + eval "$(nix print-dev-env)" +} + +use flake From 22232267e3ff13f20b8b8e6a99ea8659f518b791 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 18:07:46 +0000 Subject: [PATCH 08/11] nix: add 'nixpkgs-fmt' pre-commit hook You must run `pre-commit install` for them them to take effect. --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..45aa747 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: 'local' + hooks: + - id: 'nixpkgs-fmt' + name: 'nixpkgs-fmt' + description: 'Format nix code with nixpkgs-fmt' + entry: 'nixpkgs-fmt' + language: 'system' + files: '\.nix$' + always_run: true From 8a655debae6b69456f8cbb5cb28c5359fd072962 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 18:11:23 +0000 Subject: [PATCH 09/11] pre-commit: add more useful hooks --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45aa747..cda3975 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,3 +8,13 @@ repos: language: 'system' files: '\.nix$' always_run: true +- repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: 'v2.3.0' + hooks: + - id: 'trailing-whitespace' + - id: 'end-of-file-fixer' + - id: 'check-yaml' +- repo: 'https://github.com/jumanjihouse/pre-commit-hooks' + rev: '2.1.4' + hooks: + - id: 'forbid-binary' From 746f2be141214de49411e311fb387487e25aa7e4 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 19:32:31 +0000 Subject: [PATCH 10/11] nix: flake: use 'nixpkgs' input for 'naersk' --- flake.lock | 20 ++++---------------- flake.nix | 5 ++++- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/flake.lock b/flake.lock index bcf4f30..c0cf2ff 100644 --- a/flake.lock +++ b/flake.lock @@ -49,7 +49,9 @@ }, "naersk": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": [ + "nixpkgs" + ] }, "locked": { "lastModified": 1614785451, @@ -67,20 +69,6 @@ } }, "nixpkgs": { - "locked": { - "lastModified": 1617038346, - "narHash": "sha256-5AVL8TxnUT/8rX1XfxK05fP+jdf20jhg5gjZ2TLo/Wc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8e71416e5d4306c3800133ca6dc2ad4469480b90", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, - "nixpkgs_2": { "locked": { "lastModified": 1616670887, "narHash": "sha256-wn+l9qJfR5sj5Gq4DheJHAcBDfOs9K2p9seW2f35xzs=", @@ -102,7 +90,7 @@ "flake-utils": "flake-utils", "mozillapkgs": "mozillapkgs", "naersk": "naersk", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs" } } }, diff --git a/flake.nix b/flake.nix index c97297a..acd7f57 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,9 @@ { inputs = { - naersk.url = "github:nmattia/naersk/master"; + naersk = { + url = "github:nmattia/naersk/master"; + inputs.nixpkgs.follows = "nixpkgs"; + }; mozillapkgs = { url = "github:mozilla/nixpkgs-mozilla"; flake = false; From e45c5f380a0e1395df4f7c330615ea53e0d61776 Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Mon, 29 Mar 2021 19:33:11 +0000 Subject: [PATCH 11/11] git: ignore 'result/' symlink from 'nix build' --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ea8c4bf..d787b70 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +/result