package commands
import (
"bytes"
"fmt"
"io"
"os"
"strings"
"sync"
"github.com/git-lfs/git-lfs/v3/errors"
"github.com/git-lfs/git-lfs/v3/filepathfilter"
"github.com/git-lfs/git-lfs/v3/git"
"github.com/git-lfs/git-lfs/v3/lfs"
"github.com/git-lfs/git-lfs/v3/tq"
"github.com/git-lfs/git-lfs/v3/tr"
"github.com/git-lfs/pktline"
"github.com/spf13/cobra"
)
const (
cleanFilterBufferCapacity = 512
smudgeFilterBufferCapacity = pktline.MaxPacketLength
)
var filterSmudgeSkip bool
func filterCommand(cmd *cobra.Command, args []string) {
requireStdin(tr.Tr.Get("This command should be run by the Git filter process"))
setupRepository()
installHooks(false)
s := git.NewFilterProcessScanner(os.Stdin, os.Stdout)
if err := s.Init(); err != nil {
ExitWithError(err)
}
caps, err := s.NegotiateCapabilities()
if err != nil {
ExitWithError(err)
}
var supportsDelay bool
for _, cap := range caps {
if cap == "capability=delay" {
supportsDelay = true
break
}
}
skip := filterSmudgeSkip || cfg.Os.Bool("GIT_LFS_SKIP_SMUDGE", false)
filter := filepathfilter.New(cfg.FetchIncludePaths(), cfg.FetchExcludePaths(), filepathfilter.GitIgnore)
ptrs := make(map[string]*lfs.Pointer)
var q *tq.TransferQueue
var malformed []string
var malformedOnWindows []string
var closeOnce *sync.Once
var available chan *tq.Transfer
gitfilter := lfs.NewGitFilter(cfg)
for s.Scan() {
var n int64
var err error
var delayed bool
var w *pktline.PktlineWriter
req := s.Request()
switch req.Header["command"] {
case "clean":
s.WriteStatus(statusFromErr(nil))
w = pktline.NewPktlineWriter(os.Stdout, cleanFilterBufferCapacity)
var ptr *lfs.Pointer
ptr, err = clean(gitfilter, w, req.Payload, req.Header["pathname"], -1)
if ptr != nil {
n = ptr.Size
}
case "smudge":
if q == nil && supportsDelay {
closeOnce = new(sync.Once)
available = make(chan *tq.Transfer)
if cfg.AutoDetectRemoteEnabled() {
newRemote := git.FirstRemoteForTreeish(req.Header["treeish"])
if newRemote != "" {
cfg.SetRemote(newRemote)
}
}
q = tq.NewTransferQueue(
tq.Download,
getTransferManifestOperationRemote("download", cfg.Remote()),
cfg.Remote(),
tq.RemoteRef(currentRemoteRef()),
tq.WithBatchSize(cfg.TransferBatchSize()),
)
go infiniteTransferBuffer(q, available)
}
w = pktline.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
if req.Header["can-delay"] == "1" {
var ptr *lfs.Pointer
n, delayed, ptr, err = delayedSmudge(gitfilter, s, w, req.Payload, q, req.Header["pathname"], skip, filter)
if delayed {
ptrs[req.Header["pathname"]] = ptr
}
} else {
s.WriteStatus(statusFromErr(nil))
from, ferr := incomingOrCached(req.Payload, ptrs[req.Header["pathname"]])
if ferr != nil {
break
}
n, err = smudge(gitfilter, w, from, req.Header["pathname"], skip, filter)
if err == nil {
delete(ptrs, req.Header["pathname"])
}
}
case "list_available_blobs":
closeOnce.Do(func() {
go q.Wait()
})
paths := pathnames(readAvailable(available, q.BatchSize()))
if len(paths) == 0 {
for path, _ := range ptrs {
paths = append(paths, fmt.Sprintf("pathname=%s", path))
}
q = nil
}
err = s.WriteList(paths)
default:
ExitWithError(errors.New(tr.Tr.Get("unknown command %q", req.Header["command"])))
}
if errors.IsNotAPointerError(err) {
malformed = append(malformed, req.Header["pathname"])
err = nil
} else if possiblyMalformedObjectSize(n) {
malformedOnWindows = append(malformedOnWindows, req.Header["pathname"])
}
var status git.FilterProcessStatus
if delayed {
status = delayedStatusFromErr(err)
} else if ferr := w.Flush(); ferr != nil {
status = statusFromErr(ferr)
} else {
status = statusFromErr(err)
}
s.WriteStatus(status)
}
if len(malformed) > 0 {
fmt.Fprintln(os.Stderr, tr.Tr.GetN(
"Encountered %d file that should have been a pointer, but wasn't:",
"Encountered %d files that should have been pointers, but weren't:",
len(malformed),
len(malformed),
))
for _, m := range malformed {
fmt.Fprintf(os.Stderr, "\t%s\n", m)
}
}
if len(malformedOnWindows) > 0 && cfg.Git.Bool("lfs.largefilewarning", !git.IsGitVersionAtLeast("2.34.0")) {
fmt.Fprintln(os.Stderr, tr.Tr.GetN(
"Encountered %d file that may not have been copied correctly on Windows:",
"Encountered %d files that may not have been copied correctly on Windows:",
len(malformedOnWindows),
len(malformedOnWindows),
))
for _, m := range malformedOnWindows {
fmt.Fprintf(os.Stderr, "\t%s\n", m)
}
fmt.Fprint(os.Stderr, "\n", tr.Tr.Get("See: `git lfs help smudge` for more details."), "\n")
}
if err := s.Err(); err != nil && err != io.EOF {
ExitWithError(err)
}
}
func infiniteTransferBuffer(q *tq.TransferQueue, available chan<- *tq.Transfer) {
watch := q.Watch()
var pending []*tq.Transfer
for {
if len(pending) > 0 {
select {
case t, ok := <-watch:
if !ok {
for _, t = range pending {
available <- t
}
close(available)
return
}
pending = append(pending, t)
case available <- pending[0]:
pending = pending[1:]
}
} else {
t, ok := <-watch
if !ok {
close(available)
return
}
select {
case available <- t:
default:
pending = append(pending, t)
}
}
}
}
func incomingOrCached(r io.Reader, ptr *lfs.Pointer) (io.Reader, error) {
buf := make([]byte, 1024)
n, err := r.Read(buf)
buf = buf[:n]
if n == 0 {
if ptr == nil {
return bytes.NewReader(buf), nil
}
return strings.NewReader(ptr.Encoded()), nil
}
if err == io.EOF {
return bytes.NewReader(buf), nil
}
return io.MultiReader(bytes.NewReader(buf), r), err
}
func readAvailable(ch <-chan *tq.Transfer, cap int) []*tq.Transfer {
ts := make([]*tq.Transfer, 0, cap)
for {
select {
case t, ok := <-ch:
if !ok {
return ts
}
ts = append(ts, t)
default:
if len(ts) > 0 {
return ts
}
t, ok := <-ch
if !ok {
return ts
}
return append(ts, t)
}
}
}
func pathnames(ts []*tq.Transfer) []string {
pathnames := make([]string, 0, len(ts))
for _, t := range ts {
pathnames = append(pathnames, fmt.Sprintf("pathname=%s", t.Name))
}
return pathnames
}
func statusFromErr(err error) git.FilterProcessStatus {
if err != nil && err != io.EOF {
return git.StatusError
}
return git.StatusSuccess
}
func delayedStatusFromErr(err error) git.FilterProcessStatus {
status := statusFromErr(err)
switch status {
case git.StatusSuccess:
return git.StatusDelay
default:
return status
}
}
func init() {
RegisterCommand("filter-process", filterCommand, func(cmd *cobra.Command) {
cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
})
}