Fix git-lfs file smudging in zone source accessors

src/libexpr/eval.cc

@@ -496,6 +496,20 @@ static std::string normalizeZonePath(std::string_view zonePath)
     return path;
 }
 
+static GitAccessorOptions makeZoneAccessorOptions(ref<GitRepo> repo, const Hash & commitHash, const std::string & zonePath)
+{
+    std::string attrFp;
+    for (auto & h : repo->getGitAttributesAlongPath(commitHash, zonePath))
+        attrFp += h.gitRev();
+    return {
+        .exportIgnore = true,
+        .smudgeLfs = true,
+        .attrCommitRev = commitHash,
+        .attrPathPrefix = zonePath,
+        .attrFingerprint = std::move(attrFp),
+    };
+}
+
 // Helper to sanitize zone path for use in store path names.
 // Store paths only allow: a-zA-Z0-9 and +-._?=
 // Replaces / with - and any other invalid chars with _
@@ -792,10 +806,9 @@ StorePath EvalState::getZoneStorePath(std::string_view zonePath)
 
     if (!settings.lazyTrees) {
         debug("getZoneStorePath: %s clean, eager copy from git (tree %s)", zonePath, treeSha.gitRev());
-        // Eager mode: immediate copy from git ODB
         auto repo = getWorldRepo();
-        // exportIgnore=true: honor .gitattributes for zone content (unlike world accessor)
-        GitAccessorOptions opts{.exportIgnore = true, .smudgeLfs = false};
+        auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1);
+        auto opts = makeZoneAccessorOptions(repo, commitHash, normalizeZonePath(zonePath));
         auto accessor = repo->getAccessor(treeSha, opts, "zone");
 
         std::string name = "zone-" + sanitizeZoneNameForStore(zonePath);
@@ -839,8 +852,8 @@ StorePath EvalState::mountZoneByTreeSha(const Hash & treeSha, std::string_view z
     // This allows concurrent mounts of different zones. Multiple threads may
     // race to mount the same zone, but we check again before inserting.
     auto repo = getWorldRepo();
-    // exportIgnore=true: honor .gitattributes for zone content (unlike world accessor)
-    GitAccessorOptions opts{.exportIgnore = true, .smudgeLfs = false};
+    auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1);
+    auto opts = makeZoneAccessorOptions(repo, commitHash, std::string(zonePath));
     auto accessor = repo->getAccessor(treeSha, opts, "zone");
 
     // Generate name from zone path (sanitized for store path requirements)
@@ -947,8 +960,9 @@ StorePath EvalState::getZoneFromCheckout(std::string_view zonePath, const boost:
 
     auto makeDirtyAccessor = [&]() -> ref<SourceAccessor> {
         auto repo = getWorldRepo();
-        auto baseAccessor = repo->getAccessor(
-            getWorldTreeSha(zone), {.exportIgnore = true, .smudgeLfs = false}, "zone");
+        auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1);
+        auto zoneOpts = makeZoneAccessorOptions(repo, commitHash, zone);
+        auto baseAccessor = repo->getAccessor(getWorldTreeSha(zone), zoneOpts, "zone");
         boost::unordered_flat_set<std::string> zoneDirtyFiles;
         if (dirtyFiles) {
             auto zonePrefix = zone + "/";

src/libfetchers/git-lfs-fetch.cc

@@ -1,6 +1,7 @@
 #include "nix/fetchers/git-lfs-fetch.hh"
 #include "nix/fetchers/git-utils.hh"
 #include "nix/store/filetransfer.hh"
+#include "nix/util/base-n.hh"
 #include "nix/util/processes.hh"
 #include "nix/util/url.hh"
 #include "nix/util/users.hh"
@@ -52,6 +53,14 @@ struct LfsApiInfo
 
 } // namespace
 
+static std::string lfsEndpointUrl(const ParsedURL & url)
+{
+    auto endpoint = url.to_string();
+    if (!endpoint.ends_with(".git"))
+        endpoint += ".git";
+    return endpoint + "/info/lfs";
+}
+
 static LfsApiInfo getLfsApi(const ParsedURL & url)
 {
     assert(url.authority.has_value());
@@ -88,8 +97,38 @@ static LfsApiInfo getLfsApi(const ParsedURL & url)
 
         return {queryResp.at("href").get<std::string>(), authIt->get<std::string>()};
     }
+    else {
+        std::ostringstream inputCredDescr;
+        inputCredDescr << "protocol=" << url.scheme << "\n";
+        inputCredDescr << "host=" << url.authority->host << "\n";
+        inputCredDescr << "path=" << url.renderPath(true) << "\n";
+
+        auto [status, output] = runProgram({.program = "git", .args = {"credential", "fill"}, .input = std::move(inputCredDescr).str()});
+
+        if (output.empty())
+            throw Error("git-credential-fill: no output (cmd: 'git credential fill' for protocol=%s, host=%s, path=%s)", url.scheme, url.authority->host, url.renderPath(true));
+
+        std::string username;
+        std::string password;
+        for (auto & line : tokenizeString<Strings>(output, "\n")) {
+            auto eq = line.find('=');
+            if (eq == std::string::npos)
+                continue;
+            auto key = line.substr(0, eq);
+            auto val = line.substr(eq + 1);
+            if (key == "username")
+                username = val;
+            else if (key == "password")
+                password = val;
+        }
 
-    return {url.to_string() + "/info/lfs", std::nullopt};
+        if (username.empty() || password.empty())
+            throw Error("git-credential-fill: no credentials returned (cmd: 'git credential fill' for protocol=%s, host=%s, path=%s)", url.scheme, url.authority->host, url.renderPath(true));
+
+        return {lfsEndpointUrl(url), "Basic " + base64::encode(std::as_bytes(std::span<const char>{username + ":" + password}))};
+    }
+
+    return {lfsEndpointUrl(url), std::nullopt};
 }
 
 typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
@@ -173,10 +212,11 @@ static std::optional<Pointer> parseLfsPointer(std::string_view content, std::str
     return std::make_optional(Pointer{oid, std::stoul(size)});
 }
 
-Fetch::Fetch(git_repository * repo, git_oid rev)
+Fetch::Fetch(git_repository * repo, git_oid rev, std::string attrPathPrefix)
 {
     this->repo = repo;
     this->rev = rev;
+    this->attrPathPrefix = std::move(attrPathPrefix);
 
     const auto remoteUrl = lfs::getLfsEndpointUrl(repo);
 
@@ -189,9 +229,10 @@ bool Fetch::shouldFetch(const CanonPath & path) const
     git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
     opts.attr_commit_id = this->rev;
     opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM;
-    if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, path.rel_c_str(), "filter"))
+    auto fullPath = attrPathPrefix.empty() ? path : CanonPath("/" + attrPathPrefix) / path;
+    if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, fullPath.rel_c_str(), "filter"))
         throw Error("cannot get git-lfs attribute: %s", git_error_last()->message);
-    debug("Git filter for '%s' is '%s'", path, attr ? attr : "null");
+    debug("Git filter for '%s' is '%s'", fullPath, attr ? attr : "null");
     return attr != nullptr && !std::string(attr).compare("lfs");
 }
 
@@ -268,13 +309,26 @@ void Fetch::fetch(
         return;
     }
 
+    // Check the local git LFS object store before hitting the network
+    auto gitDir = std::filesystem::path(git_repository_commondir(repo));
+    auto localLfsPath = gitDir / "lfs" / "objects" / pointer->oid.substr(0, 2) / pointer->oid.substr(2, 2) / pointer->oid;
+    if (pathExists(localLfsPath)) {
+        debug("using local git lfs object %s", localLfsPath);
+        auto localContent = readFile(localLfsPath);
+        sizeCallback(localContent.length());
+        sink(localContent);
+        return;
+    }
+
     std::filesystem::path cacheDir = getCacheDir() / "git-lfs";
     std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
                       + "/" + pointer->oid;
     std::filesystem::path cachePath = cacheDir / key;
     if (pathExists(cachePath)) {
         debug("using cache entry %s -> %s", key, cachePath);
-        sink(readFile(cachePath));
+        auto cacheContent = readFile(cachePath);
+        sizeCallback(cacheContent.length());
+        sink(cacheContent);
         return;
     }
     debug("did not find cache entry for %s", key);

src/libfetchers/git-utils.cc

@@ -630,6 +630,38 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return toHash(*git_object_id(tree.get()));
     }
 
+    std::vector<Hash> getGitAttributesAlongPath(const Hash & commitSha, const std::string & path) override
+    {
+        std::vector<Hash> result;
+        auto oid = hashToOID(getCommitTree(commitSha));
+
+        typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
+        Tree tree;
+        if (git_tree_lookup(Setter(tree), *this, &oid))
+            throw Error("looking up tree: %s", git_error_last()->message);
+
+        auto checkAttrs = [&] {
+            auto e = git_tree_entry_byname(tree.get(), ".gitattributes");
+            if (e && git_tree_entry_type(e) == GIT_OBJECT_BLOB)
+                result.push_back(toHash(*git_tree_entry_id(e)));
+        };
+
+        checkAttrs();
+
+        for (auto & component : tokenizeString<std::vector<std::string>>(path, "/")) {
+            auto e = git_tree_entry_byname(tree.get(), component.c_str());
+            if (!e || git_tree_entry_type(e) != GIT_OBJECT_TREE)
+                break;
+            Tree next;
+            if (git_tree_lookup(Setter(next), *this, git_tree_entry_id(e)))
+                break;
+            tree = std::move(next);
+            checkAttrs();
+        }
+
+        return result;
+    }
+
     /**
      * A 'GitSourceAccessor' with no regard for export-ignore.
      */
@@ -799,7 +831,8 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti
 
 std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const
 {
-    return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "");
+    return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "")
+        + (attrFingerprint.empty() ? "" : ";af:" + attrFingerprint);
 }
 
 /**
@@ -821,7 +854,9 @@ struct GitSourceAccessor : SourceAccessor
         : state_{State{
               .repo = repo_,
               .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
-              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_,
+                  options.attrCommitRev ? hashToOID(*options.attrCommitRev) : hashToOID(rev),
+                  options.attrPathPrefix)) : std::nullopt,
               .options = options,
           }}
     {
@@ -1092,8 +1127,9 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
 {
     ref<GitRepoImpl> repo;
     std::optional<Hash> rev;
+    std::string attrPathPrefix;
 
-    GitExportIgnoreSourceAccessor(ref<GitRepoImpl> repo, ref<SourceAccessor> next, std::optional<Hash> rev)
+    GitExportIgnoreSourceAccessor(ref<GitRepoImpl> repo, ref<SourceAccessor> next, std::optional<Hash> rev, std::string attrPathPrefix = "")
         : CachingFilteringSourceAccessor(
               next,
               [&](const CanonPath & path) {
@@ -1102,12 +1138,14 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
               })
         , repo(repo)
         , rev(rev)
+        , attrPathPrefix(std::move(attrPathPrefix))
     {
     }
 
     bool gitAttrGet(const CanonPath & path, const char * attrName, const char *& valueOut)
     {
-        const char * pathCStr = path.rel_c_str();
+        auto fullPath = attrPathPrefix.empty() ? path : CanonPath("/" + attrPathPrefix) / path;
+        const char * pathCStr = fullPath.rel_c_str();
 
         if (rev) {
             git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
@@ -1457,9 +1495,10 @@ GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, s
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
-    if (options.exportIgnore)
-        return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
-    else
+    if (options.exportIgnore) {
+        auto commitRev = options.attrCommitRev ? options.attrCommitRev : std::optional(rev);
+        return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, commitRev, options.attrPathPrefix);
+    } else
         return rawGitAccessor;
 }
 
@@ -1475,7 +1514,7 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
     if (options.exportIgnore)
-        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
+        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt, options.attrPathPrefix);
     return fileAccessor;
 }
 

src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh

@@ -24,16 +24,12 @@ struct Pointer
 
 struct Fetch
 {
-    // Reference to the repository
     const git_repository * repo;
-
-    // Git commit being fetched
     git_oid rev;
-
-    // derived from git remote url
     nix::ParsedURL url;
+    std::string attrPathPrefix;
 
-    Fetch(git_repository * repo, git_oid rev);
+    Fetch(git_repository * repo, git_oid rev, std::string attrPathPrefix = "");
     bool shouldFetch(const CanonPath & path) const;
     void fetch(
         const std::string & content,

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -28,6 +28,25 @@ struct GitAccessorOptions
     bool smudgeLfs = false;
     bool submodules = false; // Currently implemented in GitInputScheme rather than GitAccessor
 
+    /**
+     * Commit OID for git_attr_get_ext with GIT_ATTR_CHECK_INCLUDE_COMMIT.
+     * Required when the accessor is created from a tree SHA rather than a
+     * commit SHA.
+     */
+    std::optional<Hash> attrCommitRev;
+
+    /**
+     * Repo-relative path prefix prepended to paths before git_attr_get_ext
+     * so libgit2 walks .gitattributes at the correct tree locations.
+     */
+    std::string attrPathPrefix;
+
+    /**
+     * Fingerprint of .gitattributes blobs along attrPathPrefix.
+     * Pre-computed by the caller so makeFingerprint doesn't need repo access.
+     */
+    std::string attrFingerprint;
+
     std::string makeFingerprint(const Hash & rev) const;
 };
 
@@ -110,6 +129,9 @@ struct GitRepo
     /** Get the root tree SHA from a commit SHA */
     virtual Hash getCommitTree(const Hash & commitSha) = 0;
 
+    /** Blob OIDs of .gitattributes at each directory from root to `path`. */
+    virtual std::vector<Hash> getGitAttributesAlongPath(const Hash & commitSha, const std::string & path) = 0;
+
     virtual ref<SourceAccessor>
     getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;