From 0db31256012e797650792141f5f61bbe4a555fe2 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 23:10:52 +0200 Subject: [PATCH 01/17] README: adjust indentation --- README.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.org b/README.org index 6d38988..627a52b 100644 --- a/README.org +++ b/README.org @@ -9,19 +9,19 @@ GitLab, for backup and visibility purposes. GitLab has a mirroring setting, but it doesn't allow for multiple mirrors, as far as I know. I also wanted my instance to be the single source of truth. -* How it works +** How it works Gitea is setup to send webhooks to my =lohr= server on every push update. When =lohr= receives a push, it clones the concerned repository, or updates it if already cloned. Then it pushes the update to *all remotes listed* in the [[file:.lohr][.lohr]] file at the repo root. -** Destructive +*** Destructive This is a very destructive process: anything removed from the single source of truth is effectively removed from any mirror as well. -* Why lohr? +** Why lohr? I was looking for a cool name, and thought about the Magic Mirror in Snow White. Some *[[https://en.wikipedia.org/wiki/Magic_Mirror_(Snow_White)][furious wikipedia searching]]* later, I found that the Magic Mirror was From 9b8668138a0b50cd6260983956704bd833d092a2 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 23:11:13 +0200 Subject: [PATCH 02/17] README: add contributing guidelines --- README.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.org b/README.org index 627a52b..eda3e36 100644 --- a/README.org +++ b/README.org @@ -21,6 +21,11 @@ file at the repo root. This is a very destructive process: anything removed from the single source of truth is effectively removed from any mirror as well. +** Contributing + +I accept patches anywhere! Feel free to [[https://github.com/alarsyo/lohr/pulls][open a GitHub Pull Request]], [[https://gitlab.com/alarsyo/lohr/-/merge_requests][a GitLab +Merge Request]], or [[https://lists.sr.ht/~alarsyo/lohr-dev][send me a patch by email]]! + ** Why lohr? I was looking for a cool name, and thought about the Magic Mirror in Snow White. From 315dc55ca745cbf7cf1be07686569358a28a382a Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Mon, 29 Mar 2021 23:17:17 +0200 Subject: [PATCH 03/17] README: add license notice --- LICENSE-APACHE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE-MIT | 19 +++++ README.org | 7 ++ 3 files changed, 227 insertions(+) create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..bec4d5f --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2021 Antoine Martin + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..02a1566 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) +Copyright (c) 2021 Antoine Martin + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.org b/README.org index eda3e36..e56966a 100644 --- a/README.org +++ b/README.org @@ -32,3 +32,10 @@ I was looking for a cool name, and thought about the Magic Mirror in Snow White. Some *[[https://en.wikipedia.org/wiki/Magic_Mirror_(Snow_White)][furious wikipedia searching]]* later, I found that the Magic Mirror was probably inspired by [[http://spessartmuseum.de/seiten/schneewittchen_engl.html][the Talking Mirror in Lohr am Main]]. That's it, that's the story. + +** License + +=lohr= is distributed under the terms of both the MIT license and the Apache +License (Version 2.0). + +See [[file:LICENSE-APACHE][LICENSE-APACHE]] and [[file:LICENSE-MIT][LICENSE-MIT]] for details. From 515ddc313016821b7253eeedc23562e88657c3d9 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 00:17:29 +0200 Subject: [PATCH 04/17] ci: setup github workflow --- .github/workflows/ci.yaml | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..0753cdc --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,69 @@ +on: [push, pull_request] + +name: CI + +jobs: + check: + name: Check + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Install nightly toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + + - name: Run cargo check + uses: actions-rs/cargo@v1 + with: + command: check + + test: + name: Test Suite + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Install nightly toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + + - name: Run cargo test + uses: actions-rs/cargo@v1 + with: + command: test + + lints: + name: Lints + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Install nightly toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt, clippy + + - name: Run cargo fmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: -- -D warnings From 9c5453bd9c0c500cfdb78c8f969e04ad89e99da1 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 00:23:09 +0200 Subject: [PATCH 05/17] ci: use clippy annotations --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0753cdc..a88c608 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -63,7 +63,7 @@ jobs: args: --all -- --check - name: Run cargo clippy - uses: actions-rs/cargo@v1 + uses: actions-rs/clippy-check@v1 with: - command: clippy - args: -- -D warnings + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features From 230f3f61768561ba9fa2709ac2af6475bf561308 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 00:55:07 +0200 Subject: [PATCH 06/17] README: write setup documentation --- README.org | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/README.org b/README.org index e56966a..cbe07d6 100644 --- a/README.org +++ b/README.org @@ -21,6 +21,68 @@ file at the repo root. This is a very destructive process: anything removed from the single source of truth is effectively removed from any mirror as well. +** Setup + +*** Quickstart + +Setting up =lohr= should be quite simple: + +1. Create a =Rocket.toml= file and [[https://rocket.rs/v0.4/guide/configuration/][add your configuration]]. + +2. Run =lohr=: + + #+begin_src sh + $ cargo run # or `cargo run --release` for production usage + #+end_src + +3. Configure your favorite git server to send a webhook to =lohr='s address on + every push event. + + I used [[https://docs.gitea.io/en-us/webhooks/][Gitea's webhooks format]], but I *think* they're similar to GitHub and + GitLab's webhooks, so these should work too! (If they don't, *please* file an + issue!) + +4. Add a =.lohr= file containing the remotes you want to mirror this repo to: + + #+begin_example + git@github.com:you/your_repo + #+end_example + + and push it. That's it! =lohr= is mirroring your repo now. + +*** Configuration + +**** Home directory + +=lohr= needs a place to clone repos and store its data. By default, it's the +current directory, but you can set the =LOHR_HOME= environment variable to +customize it. + +**** Extra remote configuration + +=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. This +file takes the following format: + +#+begin_src yaml +default_remotes: + - "git@github:user" + - "git@gitlab:user" + +additional_remotes: + - "git@git.sr.ht:~user" +#+end_src + +- ~default_remotes~ is a list of remotes to use if no ~.lohr~ file is found in a + repository. +- ~additional_remotes~ is a list of remotes to add in any case, whether the + original set of remotes is set via ~default_remotes~ or via a =.lohr= file. + +Both settings take as input a list of "stems", i.e. incomplete remote addresses, +to which the repo's name will be appended (so for example, if my +~default_remotes~ contains ~git@github.com:alarsyo~, and a push event webhook +is received for repository =git@gitlab.com:some/long/path/repo_name=, then the +mirror destination will be =git@github.com:alarsyo/repo_name=. + ** Contributing I accept patches anywhere! Feel free to [[https://github.com/alarsyo/lohr/pulls][open a GitHub Pull Request]], [[https://gitlab.com/alarsyo/lohr/-/merge_requests][a GitLab From 0ddcd24f33af3d8ba0d0701da3421bdfef28c816 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 01:04:15 +0200 Subject: [PATCH 07/17] lohr: v0.2.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fcd247..380bd66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "lohr" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "log 0.4.14", diff --git a/Cargo.toml b/Cargo.toml index 6aabd22..ac4f995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lohr" -version = "0.1.0" +version = "0.2.0" authors = ["Antoine Martin "] edition = "2018" license = "Apache-2.0 OR MIT" From 339c6e4f25713722a467ebf29f6b0f02710c5c16 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 01:08:55 +0200 Subject: [PATCH 08/17] cargo: add repo and homepage metadata --- Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index ac4f995..7cb92a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,8 @@ authors = ["Antoine Martin "] edition = "2018" license = "Apache-2.0 OR MIT" description = "A Git mirroring daemon" +homepage = "https://github.com/alarsyo/lohr" +repository = "https://github.com/alarsyo/lohr" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 7134b7700f3d7074d0f5b6b609bd77cedec24f3b Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 01:11:36 +0200 Subject: [PATCH 09/17] lohr: v0.2.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 380bd66..4d7e32b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "lohr" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "log 0.4.14", diff --git a/Cargo.toml b/Cargo.toml index 7cb92a9..17fbb0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lohr" -version = "0.2.0" +version = "0.2.1" authors = ["Antoine Martin "] edition = "2018" license = "Apache-2.0 OR MIT" From 7e3c8b8f28caa75d9f19843ac0ef7ad9b045f8a1 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 11:23:09 +0200 Subject: [PATCH 10/17] lohr: validate webhook signature Previously lohr was unusable in a production setting, anyone could forge a malicious webhook and either: - mirror a private repo of yours to another remote they own - wipe a repo of yours by forcing mirroring from an empty mirror This is no longer the case! --- Cargo.lock | 10 ++++ Cargo.toml | 4 ++ README.org | 14 ++++-- src/main.rs | 11 ++++- src/signature.rs | 122 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 src/signature.rs diff --git a/Cargo.lock b/Cargo.lock index 4d7e32b..4623823 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -336,6 +336,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.10.0" @@ -494,11 +500,15 @@ name = "lohr" version = "0.2.1" dependencies = [ "anyhow", + "hex", + "hmac", "log 0.4.14", "rocket", "rocket_contrib", "serde", + "serde_json", "serde_yaml", + "sha2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 17fbb0d..fac1516 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,12 @@ repository = "https://github.com/alarsyo/lohr" [dependencies] anyhow = "1.0.40" +hex = "0.4.3" +hmac = "0.10.1" log = "0.4.14" rocket = "0.4.7" rocket_contrib = { version = "0.4.7", features = [ "json" ] } serde = { version = "1.0.125", features = [ "derive" ] } +serde_json = "1.0.64" serde_yaml = "0.8.17" +sha2 = "0.9.3" diff --git a/README.org b/README.org index cbe07d6..f1b102f 100644 --- a/README.org +++ b/README.org @@ -29,20 +29,28 @@ Setting up =lohr= should be quite simple: 1. Create a =Rocket.toml= file and [[https://rocket.rs/v0.4/guide/configuration/][add your configuration]]. -2. Run =lohr=: +2. Export a secret variable: + + #+begin_src sh + $ export LOHR_SECRET=42 # please don't use this secret + #+end_src + +3. Run =lohr=: #+begin_src sh $ cargo run # or `cargo run --release` for production usage #+end_src -3. Configure your favorite git server to send a webhook to =lohr='s address on +4. Configure your favorite git server to send a webhook to =lohr='s address on every push event. I used [[https://docs.gitea.io/en-us/webhooks/][Gitea's webhooks format]], but I *think* they're similar to GitHub and GitLab's webhooks, so these should work too! (If they don't, *please* file an issue!) -4. Add a =.lohr= file containing the remotes you want to mirror this repo to: + Don't forget to set the webhook secret to the one you chose above. + +5. Add a =.lohr= file containing the remotes you want to mirror this repo to: #+begin_example git@github.com:you/your_repo diff --git a/src/main.rs b/src/main.rs index 6bb3613..4c7fc25 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,6 @@ use std::sync::{ use std::thread; use rocket::{http::Status, post, routes, State}; -use rocket_contrib::json::Json; use log::error; @@ -23,10 +22,14 @@ use job::Job; mod settings; use settings::GlobalSettings; +mod signature; +use signature::SignedJson; + struct JobSender(Mutex>); +struct Secret(String); #[post("/", data = "")] -fn gitea_webhook(payload: Json, sender: State) -> Status { +fn gitea_webhook(payload: SignedJson, sender: State) -> Status { // TODO: validate Gitea signature { @@ -66,6 +69,9 @@ fn main() -> anyhow::Result<()> { let homedir: PathBuf = homedir.into(); let homedir = homedir.canonicalize().expect("LOHR_HOME isn't valid!"); + let secret = env::var("LOHR_SECRET") + .expect("please provide a secret, otherwise anyone can send you a malicious webhook"); + let config = parse_config(homedir.clone())?; thread::spawn(move || { @@ -75,6 +81,7 @@ fn main() -> anyhow::Result<()> { rocket::ignite() .mount("/", routes![gitea_webhook]) .manage(JobSender(Mutex::new(sender))) + .manage(Secret(secret)) .launch(); Ok(()) diff --git a/src/signature.rs b/src/signature.rs new file mode 100644 index 0000000..48129de --- /dev/null +++ b/src/signature.rs @@ -0,0 +1,122 @@ +use std::{ + io::{Read, Write}, + ops::{Deref, DerefMut}, +}; + +use rocket::{ + data::{FromData, Outcome}, + http::ContentType, + State, +}; +use rocket::{ + data::{Transform, Transformed}, + http::Status, +}; +use rocket::{Data, Request}; + +use anyhow::anyhow; +use serde::Deserialize; + +use crate::Secret; + +const X_GITEA_SIGNATURE: &str = "X-Gitea-Signature"; + +fn validate_signature(secret: &str, signature: &str, data: &str) -> bool { + use hmac::{Hmac, Mac, NewMac}; + use sha2::Sha256; + + type HmacSha256 = Hmac; + + let mut mac = HmacSha256::new_varkey(secret.as_bytes()).expect("this should never fail"); + + mac.update(data.as_bytes()); + + match hex::decode(signature) { + Ok(bytes) => mac.verify(&bytes).is_ok(), + Err(_) => false, + } +} + +pub struct SignedJson(pub T); + +impl Deref for SignedJson { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl DerefMut for SignedJson { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +const LIMIT: u64 = 1 << 20; + +// This is a one to one implementation of request_contrib::Json's FromData, but with HMAC +// validation. +// +// Tracking issue for chaining Data guards to avoid this: +// https://github.com/SergioBenitez/Rocket/issues/775 +impl<'a, T> FromData<'a> for SignedJson +where + T: Deserialize<'a>, +{ + type Error = anyhow::Error; + type Owned = String; + type Borrowed = str; + + fn transform( + request: &Request, + data: Data, + ) -> rocket::data::Transform> { + let size_limit = request.limits().get("json").unwrap_or(LIMIT); + let mut s = String::with_capacity(512); + match data.open().take(size_limit).read_to_string(&mut s) { + Ok(_) => Transform::Borrowed(Outcome::Success(s)), + Err(e) => Transform::Borrowed(Outcome::Failure(( + Status::BadRequest, + anyhow!("couldn't read json: {}", e), + ))), + } + } + + fn from_data(request: &Request, o: Transformed<'a, Self>) -> Outcome { + let json_ct = ContentType::new("application", "json"); + if request.content_type() != Some(&json_ct) { + return Outcome::Failure((Status::BadRequest, anyhow!("wrong content type"))); + } + + let signatures = request.headers().get(X_GITEA_SIGNATURE).collect::>(); + if signatures.len() != 1 { + return Outcome::Failure(( + Status::BadRequest, + anyhow!("request header needs exactly one signature"), + )); + } + + let signature = signatures[0]; + + let content = o.borrowed()?; + + let secret = request.guard::>().unwrap(); + + if !validate_signature(&secret.0, &signature, content) { + return Outcome::Failure((Status::BadRequest, anyhow!("couldn't verify signature"))); + } + + let content = match serde_json::from_str(content) { + Ok(content) => content, + Err(e) => { + return Outcome::Failure(( + Status::BadRequest, + anyhow!("couldn't parse json: {}", e), + )) + } + }; + + Outcome::Success(SignedJson(content)) + } +} From 682e9c5bcc9070c81156667c96f06544ed98f29b Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 11:31:26 +0200 Subject: [PATCH 11/17] lohr: v0.3.0 This brings support for webhook signature verification, making lohr usable in a production setting. --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4623823..72a6705 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -497,7 +497,7 @@ dependencies = [ [[package]] name = "lohr" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "hex", diff --git a/Cargo.toml b/Cargo.toml index fac1516..896e01a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lohr" -version = "0.2.1" +version = "0.3.0" authors = ["Antoine Martin "] edition = "2018" license = "Apache-2.0 OR MIT" From ade177de543b713eaf6ffad94e838293a901b16d Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 11:38:58 +0200 Subject: [PATCH 12/17] README: mention shared secret in configuration --- README.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.org b/README.org index f1b102f..8231f47 100644 --- a/README.org +++ b/README.org @@ -66,6 +66,11 @@ Setting up =lohr= should be quite simple: current directory, but you can set the =LOHR_HOME= environment variable to customize it. +**** Shared secret + +As shown in the quickstart guide, you *must* set the =LOHR_SECRET= environment +variable. + **** Extra remote configuration =lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. This From fea9d61d2b76d6ef7299a3441aca78b3e9d362b4 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 11:41:16 +0200 Subject: [PATCH 13/17] signature: fix unused trait warning --- src/signature.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/signature.rs b/src/signature.rs index 48129de..c917db6 100644 --- a/src/signature.rs +++ b/src/signature.rs @@ -1,5 +1,5 @@ use std::{ - io::{Read, Write}, + io::Read, ops::{Deref, DerefMut}, }; From cc75f90a9ad00e141164ad783f5e97a30a31b173 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 11:47:14 +0200 Subject: [PATCH 14/17] ci: fail on clippy warnings --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a88c608..d450a8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,4 +66,4 @@ jobs: uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features + args: --all-features -- -D warnings From 1be64262b2b27a6a4ce20ee3b97bb2e0509966d7 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 14:45:43 +0200 Subject: [PATCH 15/17] main: remove solved TODO item --- src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4c7fc25..e71ec3d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,8 +30,6 @@ struct Secret(String); #[post("/", data = "")] fn gitea_webhook(payload: SignedJson, sender: State) -> Status { - // TODO: validate Gitea signature - { let sender = sender.0.lock().unwrap(); let repo = &payload.repository; From 209e1e58c7015dc2e503a4c096dc5a3bc840931e Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 30 Mar 2021 15:25:26 +0200 Subject: [PATCH 16/17] job: filter out empty lines from .lohr file --- src/job.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index 1a15233..2e91cf1 100644 --- a/src/job.rs +++ b/src/job.rs @@ -125,7 +125,13 @@ impl Job { let output = String::from_utf8(output.stdout)?; - Ok(Some(output.lines().map(String::from).collect())) + Ok(Some( + output + .lines() + .map(String::from) + .filter(|s| !s.is_empty()) + .collect(), + )) } fn get_remotes(&self, config: &GlobalSettings) -> anyhow::Result> { From 512e6cc4d1544b47134a665c95dbf1e6fbe5352a Mon Sep 17 00:00:00 2001 From: Bruno BELANYI Date: Tue, 30 Mar 2021 16:11:36 +0000 Subject: [PATCH 17/17] job: add processing blacklist to global config --- src/job.rs | 7 +++++++ src/settings.rs | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/src/job.rs b/src/job.rs index 2e91cf1..0477704 100644 --- a/src/job.rs +++ b/src/job.rs @@ -190,6 +190,13 @@ impl Job { } pub(crate) fn run(&mut self, homedir: &Path, config: &GlobalSettings) -> anyhow::Result<()> { + if config + .blacklist + .iter() + .any(|re| re.is_match(&self.repo.full_name)) + { + return Ok(()); + } let local_path = homedir.join(&self.repo.full_name); assert!(local_path.is_absolute()); self.local_path = Some(local_path); diff --git a/src/settings.rs b/src/settings.rs index bfe7744..976264f 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -10,4 +10,8 @@ pub(crate) struct GlobalSettings { /// List of remote stems to use for every repository #[serde(default)] pub additional_remotes: Vec, + /// List of regexes, if a repository's name matches any of the, it is not mirrored by `lohr` + /// even if it contains a `.lorh` file. + #[serde(with = "serde_regex")] + pub blacklist: Vec, }