fs: specify a file as existing if it's empty

If the object has a size of zero, then we already have its object: it's
the empty file, and we shouldn't need to download it from anywhere.  If
we inquire if it exists, then say it does, and say its location is the
system's equivalent of /dev/null.

The only tricky case is if we're de-duplicating, linking, or writing to
a file, in which case we should not try to use a link or replace on the
file, since we neither want to link nor replace the system's /dev/null.

Author: bk2204

commands/command_dedup.go

@@ -112,6 +112,10 @@ func dedup(p *lfs.WrappedPointer) (success bool, err error) {
 
 	// Do clone
 	srcFile := cfg.Filesystem().ObjectPathname(p.Oid)
+	if srcFile == os.DevNull {
+		return true, nil
+	}
+
 	dstFile := filepath.Join(cfg.LocalWorkingDir(), p.Name)
 
 	// Clone the file. This overwrites the destination if it exists.

commands/command_fsck.go

@@ -107,7 +107,11 @@ func fsckCommand(cmd *cobra.Command, args []string) {
 
 	for _, oid := range corruptOids {
 		badFile := filepath.Join(badDir, oid)
-		if err := os.Rename(cfg.Filesystem().ObjectPathname(oid), badFile); err != nil {
+		srcFile := cfg.Filesystem().ObjectPathname(oid)
+		if srcFile == os.DevNull {
+			continue
+		}
+		if err := os.Rename(srcFile, badFile); err != nil {
 			ExitWithError(err)
 		}
 	}

commands/command_prune.go

@@ -307,6 +307,9 @@ func pruneDeleteFiles(prunableObjects []string, logger *tasklog.Logger) {
 			problems.WriteString(fmt.Sprintf("Unable to find media path for %v: %v\n", oid, err))
 			continue
 		}
+		if mediaFile == os.DevNull {
+			continue
+		}
 		err = os.Remove(mediaFile)
 		if err != nil {
 			problems.WriteString(fmt.Sprintf("Failed to remove file %v: %v\n", mediaFile, err))

fs/fs.go

@@ -3,6 +3,8 @@ package fs
 import (
 	"bufio"
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -16,7 +18,10 @@ import (
 	"github.com/rubyist/tracerx"
 )
 
-var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)
+var (
+	oidRE             = regexp.MustCompile(`\A[[:alnum:]]{64}`)
+	EmptyObjectSHA256 = hex.EncodeToString(sha256.New().Sum(nil))
+)
 
 // Environment is a copy of a subset of the interface
 // github.com/git-lfs/git-lfs/config.Environment.
@@ -61,13 +66,19 @@ func (f *Filesystem) EachObject(fn func(Object) error) error {
 }
 
 func (f *Filesystem) ObjectExists(oid string, size int64) bool {
+	if size == 0 {
+		return true
+	}
 	return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
 }
 
 func (f *Filesystem) ObjectPath(oid string) (string, error) {
 	if len(oid) < 4 {
 		return "", fmt.Errorf("too short object ID: %q", oid)
 	}
+	if oid == EmptyObjectSHA256 {
+		return os.DevNull, nil
+	}
 	dir := f.localObjectDir(oid)
 	if err := tools.MkdirAll(dir, f); err != nil {
 		return "", fmt.Errorf("error trying to create local storage directory in %q: %s", dir, err)
@@ -76,6 +87,9 @@ func (f *Filesystem) ObjectPath(oid string) (string, error) {
 }
 
 func (f *Filesystem) ObjectPathname(oid string) string {
+	if oid == EmptyObjectSHA256 {
+		return os.DevNull
+	}
 	return filepath.Join(f.localObjectDir(oid), oid)
 }
 

lfs/pointer.go

@@ -3,8 +3,6 @@ package lfs
 import (
 	"bufio"
 	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -14,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/git-lfs/git-lfs/v3/errors"
+	"github.com/git-lfs/git-lfs/v3/fs"
 	"github.com/git-lfs/gitobj/v2"
 )
 
@@ -82,8 +81,7 @@ func (p *Pointer) Encoded() string {
 }
 
 func EmptyPointer() *Pointer {
-	oid := hex.EncodeToString(sha256.New().Sum(nil))
-	return NewPointer(oid, 0, nil)
+	return NewPointer(fs.EmptyObjectSHA256, 0, nil)
 }
 
 func EncodePointer(writer io.Writer, pointer *Pointer) (int, error) {

t/t-fetch.sh

@@ -71,6 +71,23 @@ begin_test "fetch"
 )
 end_test
 
+begin_test "fetch (empty file)"
+(
+  set -e
+  cd clone
+  rm -rf .git/lfs/objects
+
+  touch empty.dat
+  git add empty.dat
+  git commit -m 'empty'
+
+  git lfs fetch
+
+  git lfs fsck 2>&1 | tee fsck.log
+  grep "Git LFS fsck OK" fsck.log
+)
+end_test
+
 begin_test "fetch (shared repository)"
 (
   set -e

t/t-prune.sh

@@ -1038,6 +1038,27 @@ begin_test "prune --force"
 )
 end_test
 
+begin_test "prune does not fail on empty files"
+(
+  set -e
+
+  reponame="prune-empty-file"
+  setup_remote_repo "remote-$reponame"
+
+  clone_repo "remote-$reponame" "clone-$reponame"
+
+  git lfs track "*.dat" 2>&1 | tee track.log
+  grep "Tracking \"\*.dat\"" track.log
+
+  touch empty.dat
+  git add .gitattributes empty.dat
+  git commit -m 'Add empty'
+  git push origin main
+
+  git lfs prune --force
+)
+end_test
+
 begin_test "prune does not invoke external diff programs"
 (
   set -e

t/t-pull.sh

@@ -138,6 +138,14 @@ begin_test "pull"
   git lfs pull -I "*.dat"
   assert_clean_status
 
+  echo "lfs pull with empty file"
+  touch empty.dat
+  git add empty.dat
+  git commit -m 'empty'
+  git lfs pull
+  [ -z "$(cat empty.dat)" ]
+  assert_clean_status
+
   echo "lfs pull in subdir"
   cd dir
   git lfs pull