package walk
import (
"context"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/common"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/logs"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/model"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/filter"
)
type ExtractFileFilter func(relpath string) bool
type WalkFileFunc func(parent *model.File, files []*model.File)
func Walk(ctx context.Context, name, origin string, filter ExtractFileFilter, ignore ExtractFileFilter, do WalkFileFunc) (size int64, err error) {
delete, file, err := download(origin)
if err != nil {
return
}
logs.Debugf("walk %s", file)
defer func() {
if delete == "" {
return
}
logs.Debugf("remove %s", delete)
os.RemoveAll(delete)
}()
if f, xerr := os.Stat(file); xerr == nil {
if !f.IsDir() {
size = f.Size()
}
} else {
logs.Error(xerr)
err = xerr
return
}
parent := model.NewFile(file, name)
wg := &sync.WaitGroup{}
err = walk(ctx, wg, parent, filter, ignore, do)
wg.Wait()
return
}
func walk(ctx context.Context, wg *sync.WaitGroup, parent *model.File, filterFunc, ignoreFunc ExtractFileFilter, walkFunc WalkFileFunc) error {
var files []*model.File
err := filepath.Walk(parent.Abspath(), func(path string, info fs.FileInfo, err error) error {
select {
case <-ctx.Done():
return nil
default:
}
if err != nil {
logs.Warn(err)
return nil
}
rel := filepath.Join(parent.Relpath(), strings.TrimPrefix(path, parent.Abspath()))
if path != parent.Abspath() && ignoreFunc != nil && ignoreFunc(rel) {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
if strings.HasSuffix(path, ".git") || strings.HasSuffix(path, ".opensca-cache") || strings.HasSuffix(path, ".temp") {
return filepath.SkipDir
}
return nil
}
if filterFunc != nil && !filterFunc(rel) {
return nil
}
if !filter.CompressFile(rel) {
logs.Debugf("find %s", rel)
files = append(files, model.NewFile(path, rel))
return nil
}
decompress(ctx, path, filterFunc, func(dir string) {
wg.Add(1)
go func() {
defer wg.Done()
defer os.RemoveAll(dir)
parent := model.NewFile(dir, rel)
if err := walk(ctx, wg, parent, filterFunc, ignoreFunc, walkFunc); err != nil {
logs.Warn(err)
}
}()
})
return nil
})
walkFunc(parent, files)
return err
}
func decompress(ctx context.Context, input string, filter ExtractFileFilter, do func(tmpdir string)) {
tmp := common.MkdirTemp("decompress")
ok := false ||
xzip(ctx, filter, input, tmp) ||
xjar(ctx, filter, input, tmp) ||
xrar(ctx, filter, input, tmp) ||
xtar(ctx, filter, input, tmp) ||
xgz(input, tmp) ||
xbz2(input, tmp) ||
false
if ok {
do(tmp)
} else {
os.RemoveAll(tmp)
}
}