diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index d450a8b..0000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,69 +0,0 @@ -on: [push, pull_request] - -name: CI - -jobs: - check: - name: Check - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install nightly toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - - name: Run cargo check - uses: actions-rs/cargo@v1 - with: - command: check - - test: - name: Test Suite - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install nightly toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - - name: Run cargo test - uses: actions-rs/cargo@v1 - with: - command: test - - lints: - name: Lints - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install nightly toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - components: rustfmt, clippy - - - name: Run cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - - name: Run cargo clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features -- -D warnings diff --git a/Cargo.lock b/Cargo.lock index 72a6705..8fcd247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -336,12 +336,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "hkdf" version = "0.10.0" @@ -497,18 +491,14 @@ dependencies = [ [[package]] name = "lohr" -version = "0.3.0" +version = "0.1.0" dependencies = [ "anyhow", - "hex", - "hmac", "log 0.4.14", "rocket", "rocket_contrib", "serde", - "serde_json", "serde_yaml", - "sha2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 896e01a..6aabd22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,23 +1,17 @@ [package] name = "lohr" -version = "0.3.0" +version = "0.1.0" authors = ["Antoine Martin "] edition = "2018" license = "Apache-2.0 OR MIT" description = "A Git mirroring daemon" -homepage = "https://github.com/alarsyo/lohr" -repository = "https://github.com/alarsyo/lohr" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1.0.40" -hex = "0.4.3" -hmac = "0.10.1" log = "0.4.14" rocket = "0.4.7" rocket_contrib = { version = "0.4.7", features = [ "json" ] } serde = { version = "1.0.125", features = [ "derive" ] } -serde_json = "1.0.64" serde_yaml = "0.8.17" -sha2 = "0.9.3" diff --git a/LICENSE-APACHE b/LICENSE-APACHE deleted file mode 100644 index bec4d5f..0000000 --- a/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright 2021 Antoine Martin - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT deleted file mode 100644 index 02a1566..0000000 --- a/LICENSE-MIT +++ /dev/null @@ -1,19 +0,0 @@ -The MIT License (MIT) -Copyright (c) 2021 Antoine Martin - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.org b/README.org index 8231f47..6d38988 100644 --- a/README.org +++ b/README.org @@ -9,108 +9,21 @@ GitLab, for backup and visibility purposes. GitLab has a mirroring setting, but it doesn't allow for multiple mirrors, as far as I know. I also wanted my instance to be the single source of truth. -** How it works +* How it works Gitea is setup to send webhooks to my =lohr= server on every push update. When =lohr= receives a push, it clones the concerned repository, or updates it if already cloned. Then it pushes the update to *all remotes listed* in the [[file:.lohr][.lohr]] file at the repo root. -*** Destructive +** Destructive This is a very destructive process: anything removed from the single source of truth is effectively removed from any mirror as well. -** Setup - -*** Quickstart - -Setting up =lohr= should be quite simple: - -1. Create a =Rocket.toml= file and [[https://rocket.rs/v0.4/guide/configuration/][add your configuration]]. - -2. Export a secret variable: - - #+begin_src sh - $ export LOHR_SECRET=42 # please don't use this secret - #+end_src - -3. Run =lohr=: - - #+begin_src sh - $ cargo run # or `cargo run --release` for production usage - #+end_src - -4. Configure your favorite git server to send a webhook to =lohr='s address on - every push event. - - I used [[https://docs.gitea.io/en-us/webhooks/][Gitea's webhooks format]], but I *think* they're similar to GitHub and - GitLab's webhooks, so these should work too! (If they don't, *please* file an - issue!) - - Don't forget to set the webhook secret to the one you chose above. - -5. Add a =.lohr= file containing the remotes you want to mirror this repo to: - - #+begin_example - git@github.com:you/your_repo - #+end_example - - and push it. That's it! =lohr= is mirroring your repo now. - -*** Configuration - -**** Home directory - -=lohr= needs a place to clone repos and store its data. By default, it's the -current directory, but you can set the =LOHR_HOME= environment variable to -customize it. - -**** Shared secret - -As shown in the quickstart guide, you *must* set the =LOHR_SECRET= environment -variable. - -**** Extra remote configuration - -=lohr= looks for a =lohr-config.yaml= file in its =LOHR_HOME= directory. This -file takes the following format: - -#+begin_src yaml -default_remotes: - - "git@github:user" - - "git@gitlab:user" - -additional_remotes: - - "git@git.sr.ht:~user" -#+end_src - -- ~default_remotes~ is a list of remotes to use if no ~.lohr~ file is found in a - repository. -- ~additional_remotes~ is a list of remotes to add in any case, whether the - original set of remotes is set via ~default_remotes~ or via a =.lohr= file. - -Both settings take as input a list of "stems", i.e. incomplete remote addresses, -to which the repo's name will be appended (so for example, if my -~default_remotes~ contains ~git@github.com:alarsyo~, and a push event webhook -is received for repository =git@gitlab.com:some/long/path/repo_name=, then the -mirror destination will be =git@github.com:alarsyo/repo_name=. - -** Contributing - -I accept patches anywhere! Feel free to [[https://github.com/alarsyo/lohr/pulls][open a GitHub Pull Request]], [[https://gitlab.com/alarsyo/lohr/-/merge_requests][a GitLab -Merge Request]], or [[https://lists.sr.ht/~alarsyo/lohr-dev][send me a patch by email]]! - -** Why lohr? +* Why lohr? I was looking for a cool name, and thought about the Magic Mirror in Snow White. Some *[[https://en.wikipedia.org/wiki/Magic_Mirror_(Snow_White)][furious wikipedia searching]]* later, I found that the Magic Mirror was probably inspired by [[http://spessartmuseum.de/seiten/schneewittchen_engl.html][the Talking Mirror in Lohr am Main]]. That's it, that's the story. - -** License - -=lohr= is distributed under the terms of both the MIT license and the Apache -License (Version 2.0). - -See [[file:LICENSE-APACHE][LICENSE-APACHE]] and [[file:LICENSE-MIT][LICENSE-MIT]] for details. diff --git a/src/job.rs b/src/job.rs index 0477704..1a15233 100644 --- a/src/job.rs +++ b/src/job.rs @@ -125,13 +125,7 @@ impl Job { let output = String::from_utf8(output.stdout)?; - Ok(Some( - output - .lines() - .map(String::from) - .filter(|s| !s.is_empty()) - .collect(), - )) + Ok(Some(output.lines().map(String::from).collect())) } fn get_remotes(&self, config: &GlobalSettings) -> anyhow::Result> { @@ -190,13 +184,6 @@ impl Job { } pub(crate) fn run(&mut self, homedir: &Path, config: &GlobalSettings) -> anyhow::Result<()> { - if config - .blacklist - .iter() - .any(|re| re.is_match(&self.repo.full_name)) - { - return Ok(()); - } let local_path = homedir.join(&self.repo.full_name); assert!(local_path.is_absolute()); self.local_path = Some(local_path); diff --git a/src/main.rs b/src/main.rs index e71ec3d..6bb3613 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use std::sync::{ use std::thread; use rocket::{http::Status, post, routes, State}; +use rocket_contrib::json::Json; use log::error; @@ -22,14 +23,12 @@ use job::Job; mod settings; use settings::GlobalSettings; -mod signature; -use signature::SignedJson; - struct JobSender(Mutex>); -struct Secret(String); #[post("/", data = "")] -fn gitea_webhook(payload: SignedJson, sender: State) -> Status { +fn gitea_webhook(payload: Json, sender: State) -> Status { + // TODO: validate Gitea signature + { let sender = sender.0.lock().unwrap(); let repo = &payload.repository; @@ -67,9 +66,6 @@ fn main() -> anyhow::Result<()> { let homedir: PathBuf = homedir.into(); let homedir = homedir.canonicalize().expect("LOHR_HOME isn't valid!"); - let secret = env::var("LOHR_SECRET") - .expect("please provide a secret, otherwise anyone can send you a malicious webhook"); - let config = parse_config(homedir.clone())?; thread::spawn(move || { @@ -79,7 +75,6 @@ fn main() -> anyhow::Result<()> { rocket::ignite() .mount("/", routes![gitea_webhook]) .manage(JobSender(Mutex::new(sender))) - .manage(Secret(secret)) .launch(); Ok(()) diff --git a/src/settings.rs b/src/settings.rs index 976264f..bfe7744 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -10,8 +10,4 @@ pub(crate) struct GlobalSettings { /// List of remote stems to use for every repository #[serde(default)] pub additional_remotes: Vec, - /// List of regexes, if a repository's name matches any of the, it is not mirrored by `lohr` - /// even if it contains a `.lorh` file. - #[serde(with = "serde_regex")] - pub blacklist: Vec, } diff --git a/src/signature.rs b/src/signature.rs deleted file mode 100644 index c917db6..0000000 --- a/src/signature.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{ - io::Read, - ops::{Deref, DerefMut}, -}; - -use rocket::{ - data::{FromData, Outcome}, - http::ContentType, - State, -}; -use rocket::{ - data::{Transform, Transformed}, - http::Status, -}; -use rocket::{Data, Request}; - -use anyhow::anyhow; -use serde::Deserialize; - -use crate::Secret; - -const X_GITEA_SIGNATURE: &str = "X-Gitea-Signature"; - -fn validate_signature(secret: &str, signature: &str, data: &str) -> bool { - use hmac::{Hmac, Mac, NewMac}; - use sha2::Sha256; - - type HmacSha256 = Hmac; - - let mut mac = HmacSha256::new_varkey(secret.as_bytes()).expect("this should never fail"); - - mac.update(data.as_bytes()); - - match hex::decode(signature) { - Ok(bytes) => mac.verify(&bytes).is_ok(), - Err(_) => false, - } -} - -pub struct SignedJson(pub T); - -impl Deref for SignedJson { - type Target = T; - - fn deref(&self) -> &T { - &self.0 - } -} - -impl DerefMut for SignedJson { - fn deref_mut(&mut self) -> &mut T { - &mut self.0 - } -} - -const LIMIT: u64 = 1 << 20; - -// This is a one to one implementation of request_contrib::Json's FromData, but with HMAC -// validation. -// -// Tracking issue for chaining Data guards to avoid this: -// https://github.com/SergioBenitez/Rocket/issues/775 -impl<'a, T> FromData<'a> for SignedJson -where - T: Deserialize<'a>, -{ - type Error = anyhow::Error; - type Owned = String; - type Borrowed = str; - - fn transform( - request: &Request, - data: Data, - ) -> rocket::data::Transform> { - let size_limit = request.limits().get("json").unwrap_or(LIMIT); - let mut s = String::with_capacity(512); - match data.open().take(size_limit).read_to_string(&mut s) { - Ok(_) => Transform::Borrowed(Outcome::Success(s)), - Err(e) => Transform::Borrowed(Outcome::Failure(( - Status::BadRequest, - anyhow!("couldn't read json: {}", e), - ))), - } - } - - fn from_data(request: &Request, o: Transformed<'a, Self>) -> Outcome { - let json_ct = ContentType::new("application", "json"); - if request.content_type() != Some(&json_ct) { - return Outcome::Failure((Status::BadRequest, anyhow!("wrong content type"))); - } - - let signatures = request.headers().get(X_GITEA_SIGNATURE).collect::>(); - if signatures.len() != 1 { - return Outcome::Failure(( - Status::BadRequest, - anyhow!("request header needs exactly one signature"), - )); - } - - let signature = signatures[0]; - - let content = o.borrowed()?; - - let secret = request.guard::>().unwrap(); - - if !validate_signature(&secret.0, &signature, content) { - return Outcome::Failure((Status::BadRequest, anyhow!("couldn't verify signature"))); - } - - let content = match serde_json::from_str(content) { - Ok(content) => content, - Err(e) => { - return Outcome::Failure(( - Status::BadRequest, - anyhow!("couldn't parse json: {}", e), - )) - } - }; - - Outcome::Success(SignedJson(content)) - } -}