diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f8bb6e378a50..d7c9f71cda86 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -496,6 +496,20 @@ static std::string normalizeZonePath(std::string_view zonePath) return path; } +static GitAccessorOptions makeZoneAccessorOptions(ref repo, const Hash & commitHash, const std::string & zonePath) +{ + std::string attrFp; + for (auto & h : repo->getGitAttributesAlongPath(commitHash, zonePath)) + attrFp += h.gitRev(); + return { + .exportIgnore = true, + .smudgeLfs = true, + .attrCommitRev = commitHash, + .attrPathPrefix = zonePath, + .attrFingerprint = std::move(attrFp), + }; +} + // Helper to sanitize zone path for use in store path names. // Store paths only allow: a-zA-Z0-9 and +-._?= // Replaces / with - and any other invalid chars with _ @@ -792,10 +806,9 @@ StorePath EvalState::getZoneStorePath(std::string_view zonePath) if (!settings.lazyTrees) { debug("getZoneStorePath: %s clean, eager copy from git (tree %s)", zonePath, treeSha.gitRev()); - // Eager mode: immediate copy from git ODB auto repo = getWorldRepo(); - // exportIgnore=true: honor .gitattributes for zone content (unlike world accessor) - GitAccessorOptions opts{.exportIgnore = true, .smudgeLfs = false}; + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto opts = makeZoneAccessorOptions(repo, commitHash, normalizeZonePath(zonePath)); auto accessor = repo->getAccessor(treeSha, opts, "zone"); std::string name = "zone-" + sanitizeZoneNameForStore(zonePath); @@ -839,8 +852,8 @@ StorePath EvalState::mountZoneByTreeSha(const Hash & treeSha, std::string_view z // This allows concurrent mounts of different zones. Multiple threads may // race to mount the same zone, but we check again before inserting. auto repo = getWorldRepo(); - // exportIgnore=true: honor .gitattributes for zone content (unlike world accessor) - GitAccessorOptions opts{.exportIgnore = true, .smudgeLfs = false}; + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto opts = makeZoneAccessorOptions(repo, commitHash, std::string(zonePath)); auto accessor = repo->getAccessor(treeSha, opts, "zone"); // Generate name from zone path (sanitized for store path requirements) @@ -947,8 +960,9 @@ StorePath EvalState::getZoneFromCheckout(std::string_view zonePath, const boost: auto makeDirtyAccessor = [&]() -> ref { auto repo = getWorldRepo(); - auto baseAccessor = repo->getAccessor( - getWorldTreeSha(zone), {.exportIgnore = true, .smudgeLfs = false}, "zone"); + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto zoneOpts = makeZoneAccessorOptions(repo, commitHash, zone); + auto baseAccessor = repo->getAccessor(getWorldTreeSha(zone), zoneOpts, "zone"); boost::unordered_flat_set zoneDirtyFiles; if (dirtyFiles) { auto zonePrefix = zone + "/"; diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index e2b2c2e7dda7..0333570ea806 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/filetransfer.hh" +#include "nix/util/base-n.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" @@ -52,6 +53,14 @@ struct LfsApiInfo } // namespace +static std::string lfsEndpointUrl(const ParsedURL & url) +{ + auto endpoint = url.to_string(); + if (!endpoint.ends_with(".git")) + endpoint += ".git"; + return endpoint + "/info/lfs"; +} + static LfsApiInfo getLfsApi(const ParsedURL & url) { assert(url.authority.has_value()); @@ -88,8 +97,38 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) return {queryResp.at("href").get(), authIt->get()}; } + else { + std::ostringstream inputCredDescr; + inputCredDescr << "protocol=" << url.scheme << "\n"; + inputCredDescr << "host=" << url.authority->host << "\n"; + inputCredDescr << "path=" << url.renderPath(true) << "\n"; + + auto [status, output] = runProgram({.program = "git", .args = {"credential", "fill"}, .input = std::move(inputCredDescr).str()}); + + if (output.empty()) + throw Error("git-credential-fill: no output (cmd: 'git credential fill' for protocol=%s, host=%s, path=%s)", url.scheme, url.authority->host, url.renderPath(true)); + + std::string username; + std::string password; + for (auto & line : tokenizeString(output, "\n")) { + auto eq = line.find('='); + if (eq == std::string::npos) + continue; + auto key = line.substr(0, eq); + auto val = line.substr(eq + 1); + if (key == "username") + username = val; + else if (key == "password") + password = val; + } - return {url.to_string() + "/info/lfs", std::nullopt}; + if (username.empty() || password.empty()) + throw Error("git-credential-fill: no credentials returned (cmd: 'git credential fill' for protocol=%s, host=%s, path=%s)", url.scheme, url.authority->host, url.renderPath(true)); + + return {lfsEndpointUrl(url), "Basic " + base64::encode(std::as_bytes(std::span{username + ":" + password}))}; + } + + return {lfsEndpointUrl(url), std::nullopt}; } typedef std::unique_ptr> GitConfig; @@ -173,10 +212,11 @@ static std::optional parseLfsPointer(std::string_view content, std::str return std::make_optional(Pointer{oid, std::stoul(size)}); } -Fetch::Fetch(git_repository * repo, git_oid rev) +Fetch::Fetch(git_repository * repo, git_oid rev, std::string attrPathPrefix) { this->repo = repo; this->rev = rev; + this->attrPathPrefix = std::move(attrPathPrefix); const auto remoteUrl = lfs::getLfsEndpointUrl(repo); @@ -189,9 +229,10 @@ bool Fetch::shouldFetch(const CanonPath & path) const git_attr_options opts = GIT_ATTR_OPTIONS_INIT; opts.attr_commit_id = this->rev; opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM; - if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, path.rel_c_str(), "filter")) + auto fullPath = attrPathPrefix.empty() ? path : CanonPath("/" + attrPathPrefix) / path; + if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, fullPath.rel_c_str(), "filter")) throw Error("cannot get git-lfs attribute: %s", git_error_last()->message); - debug("Git filter for '%s' is '%s'", path, attr ? attr : "null"); + debug("Git filter for '%s' is '%s'", fullPath, attr ? attr : "null"); return attr != nullptr && !std::string(attr).compare("lfs"); } @@ -268,13 +309,26 @@ void Fetch::fetch( return; } + // Check the local git LFS object store before hitting the network + auto gitDir = std::filesystem::path(git_repository_commondir(repo)); + auto localLfsPath = gitDir / "lfs" / "objects" / pointer->oid.substr(0, 2) / pointer->oid.substr(2, 2) / pointer->oid; + if (pathExists(localLfsPath)) { + debug("using local git lfs object %s", localLfsPath); + auto localContent = readFile(localLfsPath); + sizeCallback(localContent.length()); + sink(localContent); + return; + } + std::filesystem::path cacheDir = getCacheDir() / "git-lfs"; std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false) + "/" + pointer->oid; std::filesystem::path cachePath = cacheDir / key; if (pathExists(cachePath)) { debug("using cache entry %s -> %s", key, cachePath); - sink(readFile(cachePath)); + auto cacheContent = readFile(cachePath); + sizeCallback(cacheContent.length()); + sink(cacheContent); return; } debug("did not find cache entry for %s", key); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 9e79cdbff8d3..e19d6e2f2e9f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -630,6 +630,38 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return toHash(*git_object_id(tree.get())); } + std::vector getGitAttributesAlongPath(const Hash & commitSha, const std::string & path) override + { + std::vector result; + auto oid = hashToOID(getCommitTree(commitSha)); + + typedef std::unique_ptr> Tree; + Tree tree; + if (git_tree_lookup(Setter(tree), *this, &oid)) + throw Error("looking up tree: %s", git_error_last()->message); + + auto checkAttrs = [&] { + auto e = git_tree_entry_byname(tree.get(), ".gitattributes"); + if (e && git_tree_entry_type(e) == GIT_OBJECT_BLOB) + result.push_back(toHash(*git_tree_entry_id(e))); + }; + + checkAttrs(); + + for (auto & component : tokenizeString>(path, "/")) { + auto e = git_tree_entry_byname(tree.get(), component.c_str()); + if (!e || git_tree_entry_type(e) != GIT_OBJECT_TREE) + break; + Tree next; + if (git_tree_lookup(Setter(next), *this, git_tree_entry_id(e))) + break; + tree = std::move(next); + checkAttrs(); + } + + return result; + } + /** * A 'GitSourceAccessor' with no regard for export-ignore. */ @@ -799,7 +831,8 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const { - return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : ""); + return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "") + + (attrFingerprint.empty() ? "" : ";af:" + attrFingerprint); } /** @@ -821,7 +854,9 @@ struct GitSourceAccessor : SourceAccessor : state_{State{ .repo = repo_, .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()), - .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt, + .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, + options.attrCommitRev ? hashToOID(*options.attrCommitRev) : hashToOID(rev), + options.attrPathPrefix)) : std::nullopt, .options = options, }} { @@ -1092,8 +1127,9 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor { ref repo; std::optional rev; + std::string attrPathPrefix; - GitExportIgnoreSourceAccessor(ref repo, ref next, std::optional rev) + GitExportIgnoreSourceAccessor(ref repo, ref next, std::optional rev, std::string attrPathPrefix = "") : CachingFilteringSourceAccessor( next, [&](const CanonPath & path) { @@ -1102,12 +1138,14 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor }) , repo(repo) , rev(rev) + , attrPathPrefix(std::move(attrPathPrefix)) { } bool gitAttrGet(const CanonPath & path, const char * attrName, const char *& valueOut) { - const char * pathCStr = path.rel_c_str(); + auto fullPath = attrPathPrefix.empty() ? path : CanonPath("/" + attrPathPrefix) / path; + const char * pathCStr = fullPath.rel_c_str(); if (rev) { git_attr_options opts = GIT_ATTR_OPTIONS_INIT; @@ -1457,9 +1495,10 @@ GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, s auto self = ref(shared_from_this()); ref rawGitAccessor = getRawAccessor(rev, options); rawGitAccessor->setPathDisplay(std::move(displayPrefix)); - if (options.exportIgnore) - return make_ref(self, rawGitAccessor, rev); - else + if (options.exportIgnore) { + auto commitRev = options.attrCommitRev ? options.attrCommitRev : std::optional(rev); + return make_ref(self, rawGitAccessor, commitRev, options.attrPathPrefix); + } else return rawGitAccessor; } @@ -1475,7 +1514,7 @@ ref GitRepoImpl::getAccessor( std::move(makeNotAllowedError)) .cast(); if (options.exportIgnore) - fileAccessor = make_ref(self, fileAccessor, std::nullopt); + fileAccessor = make_ref(self, fileAccessor, std::nullopt, options.attrPathPrefix); return fileAccessor; } diff --git a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh index b59da391a056..8fea1c17983c 100644 --- a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh +++ b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh @@ -24,16 +24,12 @@ struct Pointer struct Fetch { - // Reference to the repository const git_repository * repo; - - // Git commit being fetched git_oid rev; - - // derived from git remote url nix::ParsedURL url; + std::string attrPathPrefix; - Fetch(git_repository * repo, git_oid rev); + Fetch(git_repository * repo, git_oid rev, std::string attrPathPrefix = ""); bool shouldFetch(const CanonPath & path) const; void fetch( const std::string & content, diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index fd14cab555b6..ba796b952ab6 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -28,6 +28,25 @@ struct GitAccessorOptions bool smudgeLfs = false; bool submodules = false; // Currently implemented in GitInputScheme rather than GitAccessor + /** + * Commit OID for git_attr_get_ext with GIT_ATTR_CHECK_INCLUDE_COMMIT. + * Required when the accessor is created from a tree SHA rather than a + * commit SHA. + */ + std::optional attrCommitRev; + + /** + * Repo-relative path prefix prepended to paths before git_attr_get_ext + * so libgit2 walks .gitattributes at the correct tree locations. + */ + std::string attrPathPrefix; + + /** + * Fingerprint of .gitattributes blobs along attrPathPrefix. + * Pre-computed by the caller so makeFingerprint doesn't need repo access. + */ + std::string attrFingerprint; + std::string makeFingerprint(const Hash & rev) const; }; @@ -110,6 +129,9 @@ struct GitRepo /** Get the root tree SHA from a commit SHA */ virtual Hash getCommitTree(const Hash & commitSha) = 0; + /** Blob OIDs of .gitattributes at each directory from root to `path`. */ + virtual std::vector getGitAttributesAlongPath(const Hash & commitSha, const std::string & path) = 0; + virtual ref getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;