package commands
import (
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"github.com/git-lfs/git-lfs/v3/errors"
"github.com/git-lfs/git-lfs/v3/git"
"github.com/git-lfs/git-lfs/v3/lfs"
"github.com/git-lfs/git-lfs/v3/tasklog"
"github.com/git-lfs/git-lfs/v3/tools"
"github.com/git-lfs/git-lfs/v3/tq"
"github.com/git-lfs/git-lfs/v3/tr"
"github.com/rubyist/tracerx"
)
func uploadForRefUpdates(ctx *uploadContext, updates []*git.RefUpdate, pushAll bool) error {
gitscanner := ctx.buildGitScanner()
defer ctx.ReportErrors()
verifyLocksForUpdates(ctx.lockVerifier, updates)
exclude := make([]string, 0, len(updates))
for _, update := range updates {
remoteRefSha := update.RemoteRef().Sha
if update.LocalRefCommitish() != remoteRefSha {
exclude = append(exclude, remoteRefSha)
}
}
for _, update := range updates {
q := ctx.NewQueue(
tq.RemoteRef(update.RemoteRef()),
)
err := uploadRangeOrAll(gitscanner, ctx, q, exclude, update, pushAll)
ctx.CollectErrors(q)
if err != nil {
return errors.Wrap(err, tr.Tr.Get("ref %q:", update.LocalRef().Name))
}
}
return nil
}
func uploadRangeOrAll(g *lfs.GitScanner, ctx *uploadContext, q *tq.TransferQueue, exclude []string, update *git.RefUpdate, pushAll bool) error {
cb := ctx.gitScannerCallback(q)
if pushAll {
if err := g.ScanRefWithDeleted(update.LocalRefCommitish(), cb); err != nil {
return err
}
} else {
if err := g.ScanMultiRangeToRemote(update.LocalRefCommitish(), exclude, cb); err != nil {
return err
}
}
return ctx.scannerError()
}
type uploadContext struct {
Remote string
DryRun bool
Manifest tq.Manifest
uploadedOids tools.StringSet
gitfilter *lfs.GitFilter
logger *tasklog.Logger
meter *tq.Meter
committerName string
committerEmail string
lockVerifier *lockVerifier
allowMissing bool
scannerErr error
errMu sync.Mutex
missing map[string]string
corrupt map[string]string
otherErrs []error
}
func newUploadContext(dryRun bool) *uploadContext {
remote := cfg.PushRemote()
manifest := getTransferManifestOperationRemote("upload", remote)
ctx := &uploadContext{
Remote: remote,
Manifest: manifest,
DryRun: dryRun,
uploadedOids: tools.NewStringSet(),
gitfilter: lfs.NewGitFilter(cfg),
lockVerifier: newLockVerifier(manifest),
allowMissing: cfg.Git.Bool("lfs.allowincompletepush", false),
missing: make(map[string]string),
corrupt: make(map[string]string),
otherErrs: make([]error, 0),
}
var sink io.Writer = os.Stdout
if dryRun {
sink = io.Discard
}
ctx.logger = tasklog.NewLogger(sink,
tasklog.ForceProgress(cfg.ForceProgress()),
)
ctx.meter = buildProgressMeter(ctx.DryRun, tq.Upload)
ctx.logger.Enqueue(ctx.meter)
ctx.committerName, ctx.committerEmail = cfg.CurrentCommitter()
return ctx
}
func (c *uploadContext) NewQueue(options ...tq.Option) *tq.TransferQueue {
return tq.NewTransferQueue(tq.Upload, c.Manifest, c.Remote, append(options,
tq.DryRun(c.DryRun),
tq.WithProgress(c.meter),
tq.WithBatchSize(cfg.TransferBatchSize()),
)...)
}
func (c *uploadContext) scannerError() error {
c.errMu.Lock()
defer c.errMu.Unlock()
return c.scannerErr
}
func (c *uploadContext) addScannerError(err error) {
c.errMu.Lock()
defer c.errMu.Unlock()
if c.scannerErr != nil {
c.scannerErr = fmt.Errorf("%v\n%v", c.scannerErr, err)
} else {
c.scannerErr = err
}
}
func (c *uploadContext) buildGitScanner() *lfs.GitScanner {
return lfs.NewGitScannerForPush(cfg, c.Remote, func(n string) { c.lockVerifier.LockedByThem(n) }, c.lockVerifier)
}
func (c *uploadContext) gitScannerCallback(tqueue *tq.TransferQueue) func(*lfs.WrappedPointer, error) {
return func(p *lfs.WrappedPointer, err error) {
if err != nil {
c.addScannerError(err)
} else {
c.UploadPointers(tqueue, p)
}
}
}
func (c *uploadContext) SetUploaded(oid string) {
c.uploadedOids.Add(oid)
}
func (c *uploadContext) HasUploaded(oid string) bool {
return c.uploadedOids.Contains(oid)
}
func (c *uploadContext) prepareUpload(unfiltered ...*lfs.WrappedPointer) []*lfs.WrappedPointer {
numUnfiltered := len(unfiltered)
uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)
uniqOids := tools.NewStringSet()
for _, p := range unfiltered {
if uniqOids.Contains(p.Oid) || c.HasUploaded(p.Oid) || p.Size == 0 {
continue
}
uniqOids.Add(p.Oid)
var canUpload bool = true
if c.lockVerifier.LockedByThem(p.Name) {
canUpload = !c.lockVerifier.Enabled()
}
c.lockVerifier.LockedByUs(p.Name)
if canUpload {
c.meter.Add(p.Size)
uploadables = append(uploadables, p)
}
}
return uploadables
}
func (c *uploadContext) UploadPointers(q *tq.TransferQueue, unfiltered ...*lfs.WrappedPointer) {
if c.DryRun {
for _, p := range unfiltered {
if c.HasUploaded(p.Oid) {
continue
}
Print("%s %s => %s", tr.Tr.Get("push"), p.Oid, p.Name)
c.SetUploaded(p.Oid)
}
return
}
pointers := c.prepareUpload(unfiltered...)
for _, p := range pointers {
t, err := c.uploadTransfer(p)
if err != nil && !errors.IsCleanPointerError(err) {
ExitWithError(err)
}
q.Add(t.Name, t.Path, t.Oid, t.Size, t.Missing, nil)
c.SetUploaded(p.Oid)
}
}
func (c *uploadContext) CollectErrors(tqueue *tq.TransferQueue) {
tqueue.Wait()
for _, err := range tqueue.Errors() {
if malformed, ok := err.(*tq.MalformedObjectError); ok {
if malformed.Missing() {
c.missing[malformed.Name] = malformed.Oid
} else if malformed.Corrupt() {
c.corrupt[malformed.Name] = malformed.Oid
}
} else {
c.otherErrs = append(c.otherErrs, err)
}
}
}
func (c *uploadContext) ReportErrors() {
c.meter.Finish()
for _, err := range c.otherErrs {
FullError(err)
}
if len(c.missing) > 0 || len(c.corrupt) > 0 {
var action string
if c.allowMissing {
action = tr.Tr.Get("missing objects")
} else {
action = tr.Tr.Get("failed")
}
Print(tr.Tr.Get("Git LFS upload %s:", action))
for name, oid := range c.missing {
Print(tr.Tr.Get(" (missing) %s (%s)", name, oid))
}
for name, oid := range c.corrupt {
Print(tr.Tr.Get(" (corrupt) %s (%s)", name, oid))
}
if !c.allowMissing {
pushMissingHint := []string{
tr.Tr.Get("hint: Your push was rejected due to missing or corrupt local objects."),
tr.Tr.Get("hint: You can disable this check with: `git config lfs.allowincompletepush true`"),
}
Print(strings.Join(pushMissingHint, "\n"))
os.Exit(2)
}
}
if len(c.otherErrs) > 0 {
os.Exit(2)
}
if c.lockVerifier.HasUnownedLocks() {
Print(tr.Tr.Get("Unable to push locked files:"))
for _, unowned := range c.lockVerifier.UnownedLocks() {
Print("* %s - %s", unowned.Path(), unowned.Owners())
}
if c.lockVerifier.Enabled() {
Exit(tr.Tr.Get("Cannot update locked files."))
} else {
Error(tr.Tr.Get("warning: The above files would have halted this push."))
}
} else if c.lockVerifier.HasOwnedLocks() {
Print(tr.Tr.Get("Consider unlocking your own locked files: (`git lfs unlock <path>`)"))
for _, owned := range c.lockVerifier.OwnedLocks() {
Print("* %s", owned.Path())
}
}
}
var (
githubHttps, _ = url.Parse("https://github.com")
githubSsh, _ = url.Parse("ssh://github.com")
hostsWithKnownLockingSupport = []*url.URL{
githubHttps, githubSsh,
}
)
func (c *uploadContext) uploadTransfer(p *lfs.WrappedPointer) (*tq.Transfer, error) {
var missing bool
filename := p.Name
oid := p.Oid
localMediaPath, err := c.gitfilter.ObjectPath(oid)
if err != nil {
return nil, errors.Wrap(err, tr.Tr.Get("Error uploading file %s (%s)", filename, oid))
}
if len(filename) > 0 {
if missing, err = c.ensureFile(filename, localMediaPath, oid); err != nil && !errors.IsCleanPointerError(err) {
return nil, err
}
}
return &tq.Transfer{
Name: filename,
Path: localMediaPath,
Oid: oid,
Size: p.Size,
Missing: missing,
}, nil
}
func (c *uploadContext) ensureFile(smudgePath, cleanPath, oid string) (bool, error) {
if _, err := os.Stat(cleanPath); err == nil {
return false, nil
}
localPath := filepath.Join(cfg.LocalWorkingDir(), smudgePath)
file, err := os.Open(localPath)
if err != nil {
return !c.allowMissing, nil
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return false, err
}
cleaned, err := c.gitfilter.Clean(file, file.Name(), stat.Size(), nil)
if cleaned != nil {
cleaned.Teardown()
}
if err != nil {
return false, err
}
return false, nil
}
func supportsLockingAPI(rawurl string) bool {
u, err := url.Parse(rawurl)
if err != nil {
tracerx.Printf("commands: unable to parse %q to determine locking support: %v", rawurl, err)
return false
}
for _, supported := range hostsWithKnownLockingSupport {
if supported.Scheme == u.Scheme &&
supported.Hostname() == u.Hostname() &&
strings.HasPrefix(u.Path, supported.Path) {
return true
}
}
return false
}
func disableFor(rawurl string) error {
tracerx.Printf("commands: disabling lock verification for %q", rawurl)
key := strings.Join([]string{"lfs", rawurl, "locksverify"}, ".")
_, err := cfg.SetGitLocalKey(key, "false")
return err
}