From 7517c134c57c7ce967a7b603482a23f4952331a0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 00:29:23 -0500 Subject: [PATCH 1/4] flake.lock: Update Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newer 2.19-maintenance has some `restricted-eval` fixes that benefit Hydra. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-12-11) → 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..66e30323 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1702510710, + "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", + "ref": "2.19-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..8280e076 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: From abd858d3dcd8dbcb2ee2d17ff5b03dbce46821e7 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 19 Dec 2023 07:54:40 +1000 Subject: [PATCH 2/4] Document the `store_uri` parameter by way of example --- doc/manual/src/configuration.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md index 02210449..4954040c 100644 --- a/doc/manual/src/configuration.md +++ b/doc/manual/src/configuration.md @@ -74,6 +74,30 @@ following: } } +Populating a Cache +------------------ + +A common use for Hydra is to pre-build and cache derivations which +take a long time to build. While it is possible to direcly access the +Hydra server's store over SSH, a more scalable option is to upload +built derivations to a remote store like an [S3-compatible object +store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting +the `store_uri` parameter will cause Hydra to sign and upload +derivations as they are built: + +``` +store_uri = s3://cache-bucket-name?compression=zstd¶llel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key +``` + +This example uses [Zstandard](https://github.com/facebook/zstd) +compression on derivations to reduce CPU usage on the server, but +[Brotli](https://brotli.org/) compression for derivation listings and +build logs because it has better browser support. + +See [`nix help +stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) +for a description of the store URI format. + Statsd Configuration -------------------- From 75f26f1fc4f114b7f930c57ba37212ea67990de1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Dec 2023 19:10:58 -0500 Subject: [PATCH 3/4] Clean up `std::optional` dereferencing in the queue runner Instead of doing this partial operation a number of times, assert (with a comment, get a reference to the thing inside, and use that just once. (This refactor was done twice, "just once" for each time.) --- src/hydra-queue-runner/queue-monitor.cc | 26 ++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..09a2871e 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -495,12 +495,14 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); - if (/* localStore != destStore && */ localStore->isValidPath(*path)) + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*path, {}}}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -509,25 +511,27 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; try { time_t startTime = time(0); - if (localStore->isValidPath(*path)) + if (localStore->isValidPath(path)) printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); else { printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); + localStore->ensurePath(path); // FIXME: should copy directly from substituter to destStore. } copyClosure(*localStore, *destStore, - StorePathSet { *path }, + StorePathSet { path }, NoRepair, CheckSigs, NoSubstitute); time_t stopTime = time(0); @@ -535,13 +539,13 @@ Step::ptr State::createStep(ref destStore, { auto mc = startDbUpdate(); pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); txn.commit(); } } catch (Error & e) { printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath), e.what()); valid = false; From 4e8fbaa3d688a4ffa95ccbaa0af889569c918633 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 00:00:23 -0500 Subject: [PATCH 4/4] Replace `Child` with `SSHMaster::Connection` Nix defines basically an identical struct for the same purpose, so let's just use that. --- src/hydra-queue-runner/build-remote.cc | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index b117fdad..a4f3a35c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -14,19 +14,13 @@ #include "util.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" +#include "ssh.hh" #include "finally.hh" #include "url.hh" using namespace nix; -struct Child -{ - Pid pid; - AutoCloseFD to, from; -}; - - static void append(Strings & dst, const Strings & src) { dst.insert(dst.end(), src.begin(), src.end()); @@ -54,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child) +static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) { std::string pgmName; Pipe to, from; @@ -84,7 +78,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil append(argv, extraArgs); } - child.pid = startProcess([&]() { + child.sshPid = startProcess([&]() { restoreProcessContext(); if (dup2(to.readSide.get(), STDIN_FILENO) == -1) @@ -104,8 +98,8 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil to.readSide = -1; from.writeSide = -1; - child.to = to.writeSide.release(); - child.from = from.readSide.release(); + child.in = to.writeSide.release(); + child.out = from.readSide.release(); } @@ -488,13 +482,13 @@ void State::buildRemote(ref destStore, updateStep(ssConnecting); // FIXME: rewrite to use Store. - Child child; + SSHMaster::Connection child; build_remote::openConnection(machine, tmpDir, logFD.get(), child); { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child.pid; + activeStepState->pid = child.sshPid; } Finally clearPid([&]() { @@ -510,8 +504,8 @@ void State::buildRemote(ref destStore, }); Machine::Connection conn { - .from = child.from.get(), - .to = child.to.get(), + .from = child.out.get(), + .to = child.in.get(), .machine = machine, }; @@ -523,7 +517,7 @@ void State::buildRemote(ref destStore, try { build_remote::handshake(conn, buildOptions.repeats); } catch (EndOfFile & e) { - child.pid.wait(); + child.sshPid.wait(); std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } @@ -617,8 +611,8 @@ void State::buildRemote(ref destStore, } /* Shut down the connection. */ - child.to = -1; - child.pid.wait(); + child.in = -1; + child.sshPid.wait(); } catch (Error & e) { /* Disable this machine until a certain period of time has