package java
import (
"bufio"
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"github.com/xmirrorsecurity/opensca-cli/v3/cmd/config"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/common"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/logs"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/model"
"github.com/xmirrorsecurity/opensca-cli/v3/opensca/sca/cache"
)
func ParsePoms(ctx context.Context, poms []*Pom, exclusion []*Pom, call func(pom *Pom, root *model.DepGraph)) {
inheritModules(poms)
gavMap := map[string]*model.File{}
PathMap := map[string]*model.File{}
for _, pom := range poms {
gavMap[pom.GAV()] = pom.File
pom.Update(&pom.PomDependency)
gavMap[pom.GAV()] = pom.File
if pom.File.Relpath() != "" {
PathMap[pom.File.Relpath()] = pom.File
}
}
getpom := func(dep PomDependency, repos ...[]string) *Pom {
f, ok := gavMap[dep.GAV()]
if !ok && dep.RelativePath != "" && dep.Define != nil && dep.Define.File.Relpath() != "" {
pompath := filepath.Join(filepath.Dir(dep.Define.File.Relpath()), dep.RelativePath)
f, ok = PathMap[pompath]
}
var p *Pom
if ok {
f.OpenReader(func(reader io.Reader) {
p = ReadPom(reader)
p.File = f
})
}
if p != nil {
return p
}
var rs []common.RepoConfig
for _, urls := range repos {
for _, url := range urls {
rs = append(rs, common.RepoConfig{Url: url})
}
}
p = mavenOrigin(dep.GroupId, dep.ArtifactId, dep.Version, rs...)
if p == nil {
logs.Warnf("not found pom %s", dep.Index3())
}
return p
}
exclusionMap := map[*Pom]bool{}
for _, pom := range exclusion {
exclusionMap[pom] = true
}
wg := sync.WaitGroup{}
for _, pom := range poms {
if exclusionMap[pom] {
continue
}
wg.Add(1)
go func(pom *Pom) {
defer wg.Done()
call(pom, parsePom(ctx, pom, getpom))
}(pom)
}
wg.Wait()
}
func inheritModules(poms []*Pom) {
gavMap := map[string]bool{}
for _, pom := range poms {
gavMap[pom.GAV()] = true
}
_mod := model.NewDepGraphMap(nil, func(s ...string) *model.DepGraph { return &model.DepGraph{Name: s[0]} })
for _, pom := range poms {
n := _mod.LoadOrStore(pom.ArtifactId)
n.Expand = pom
for _, subMod := range pom.Modules {
n.AppendChild(_mod.LoadOrStore(subMod))
}
if gavMap[pom.Parent.GAV()] {
_mod.LoadOrStore(pom.Parent.ArtifactId).AppendChild(n)
}
}
_mod.Range(func(k string, v *model.DepGraph) bool {
if len(v.Parents) > 0 {
return true
}
v.ForEachPath(func(p, n *model.DepGraph) bool {
if n.Expand == nil {
return true
}
for _, p := range n.Parents {
if p.Expand != nil {
return true
}
}
pom, ok := n.Expand.(*Pom)
if !ok {
return true
}
n.Expand = nil
for _, c := range n.Children {
mod := _mod.LoadOrStore(c.Name)
if mod.Expand == nil {
continue
}
modpom, ok := mod.Expand.(*Pom)
if !ok {
continue
}
if modpom.Properties == nil {
modpom.Properties = PomProperties{}
}
for k, v := range pom.Properties {
if _, ok := modpom.Properties[k]; !ok {
modpom.Properties[k] = v
}
}
}
return true
})
return true
})
}
type getPomFunc func(dep PomDependency, repos ...[]string) *Pom
func inheritPom(pom *Pom, getpom getPomFunc) {
parentSet := map[string]bool{}
parent := pom.Parent
for parent.ArtifactId != "" {
pom.Update(&parent)
if parentSet[parent.Index3()] {
break
} else {
parentSet[parent.Index3()] = true
}
parentPom := getpom(parent, pom.Repositories, pom.Mirrors)
if parentPom == nil {
break
}
parentPom.PomDependency = parent
parent = parentPom.Parent
for k, v := range parentPom.Properties {
if _, ok := pom.Properties[k]; !ok {
pom.Properties[k] = v
}
}
pom.DependencyManagement = append(pom.DependencyManagement, parentPom.DependencyManagement...)
pom.Dependencies = append(pom.Dependencies, parentPom.Dependencies...)
pom.Repositories = append(pom.Repositories, parentPom.Repositories...)
pom.Mirrors = append(pom.Mirrors, parentPom.Mirrors...)
}
pom.Update(&pom.PomDependency)
pom.Update(&pom.Parent)
depIndex2Set := map[string]bool{}
for i := len(pom.Dependencies) - 1; i >= 0; i-- {
dep := pom.Dependencies[i]
if depIndex2Set[dep.Index2()] {
pom.Dependencies = append(pom.Dependencies[:i], pom.Dependencies[i+1:]...)
} else {
depIndex2Set[dep.Index2()] = true
}
}
depIndex2Set = map[string]bool{}
for i := 0; i < len(pom.DependencyManagement); {
dep := pom.DependencyManagement[i]
pom.Update(dep)
if depIndex2Set[dep.Index2()] {
pom.DependencyManagement = append(pom.DependencyManagement[:i], pom.DependencyManagement[i+1:]...)
continue
} else {
i++
depIndex2Set[dep.Index2()] = true
}
if dep.Scope != "import" {
continue
}
ipom := getpom(*dep, pom.Repositories, pom.Mirrors)
if ipom == nil {
continue
}
ipom.PomDependency = *dep
inheritPom(ipom, getpom)
for _, idep := range ipom.DependencyManagement {
if depIndex2Set[idep.Index2()] {
continue
}
ipom.Update(idep)
pom.DependencyManagement = append(pom.DependencyManagement, idep)
}
}
}
func replacePomDependency(old, new *PomDependency, indirect bool) (replaced *PomDependency) {
originVersion := old.Version
originScope := old.Scope
dep := *new
replaced = &dep
if indirect && replaced.Version == "" {
replaced.Version = originVersion
}
if !indirect && originScope != "" {
replaced.Scope = originScope
}
return
}
func parsePom(ctx context.Context, pom *Pom, getpom getPomFunc) *model.DepGraph {
if pom.Properties == nil {
pom.Properties = PomProperties{}
}
pom.Update(&pom.PomDependency)
inheritPom(pom, getpom)
rootPomManagement := map[string]*PomDependency{}
for _, dep := range pom.DependencyManagement {
if dep.Scope != "import" {
rootPomManagement[dep.Index2()] = dep
}
}
root := &model.DepGraph{Vendor: pom.GroupId, Name: pom.ArtifactId, Version: pom.Version, Path: pom.File.Relpath()}
root.Expand = pom
depIndex2Set := map[string]bool{}
root.ForEachNode(func(p, n *model.DepGraph) bool {
select {
case <-ctx.Done():
return false
default:
}
if n.Expand == nil {
return true
}
np := n.Expand.(*Pom)
for _, lic := range np.Licenses {
n.AppendLicense(lic)
}
depManagement := map[string]*PomDependency{}
for _, dep := range np.DependencyManagement {
if dep.Scope != "import" {
depManagement[dep.Index2()] = dep
}
}
for _, dep := range np.Dependencies {
if dep.Scope == "provided" || dep.Optional {
continue
}
if np != pom && dep.Scope == "test" {
continue
}
if np != pom {
if d, ok := depManagement[dep.Index2()]; ok {
if d.Optional ||
d.Scope == "provided" ||
d.Scope == "test" {
continue
}
}
}
np.Update(dep)
if d, ok := depManagement[dep.Index2()]; ok {
exclusion := append(dep.Exclusions, d.Exclusions...)
if dep.Version == "" {
dep = replacePomDependency(dep, d, false)
}
dep.Exclusions = exclusion
np.Update(dep)
}
if np != pom || dep.Version == "" {
d, ok := rootPomManagement[dep.Index2()]
if ok {
exclusion := append(dep.Exclusions, d.Exclusions...)
dep = replacePomDependency(dep, d, true)
dep.Exclusions = exclusion
pom.Update(dep)
}
}
if np.NeedExclusion(*dep) {
continue
}
if depIndex2Set[dep.Index2()] {
continue
}
depIndex2Set[dep.Index2()] = true
if dep.Check() {
logs.Debugf("find %s", dep.ImportPathStack())
} else {
logs.Warnf("find invalid %s", dep.ImportPathStack())
continue
}
sub := &model.DepGraph{Vendor: dep.GroupId, Name: dep.ArtifactId, Version: dep.Version}
sub.Develop = dep.Scope == "test"
if subpom := getpom(*dep, np.Repositories, np.Mirrors); subpom != nil {
subpom.PomDependency = *dep
subpom.Exclusions = append(subpom.Exclusions, np.Exclusions...)
inheritPom(subpom, getpom)
sub.Expand = subpom
}
n.AppendChild(sub)
}
return true
})
return root
}
var mavenOrigin = func(groupId, artifactId, version string, repos ...common.RepoConfig) *Pom {
var p *Pom
path := cache.Path(groupId, artifactId, version, model.Lan_Java)
cache.Load(path, func(reader io.Reader) {
p = ReadPom(reader)
})
if p != nil {
return p
}
DownloadPomFromRepo(PomDependency{GroupId: groupId, ArtifactId: artifactId, Version: version}, func(r io.Reader) {
data, err := io.ReadAll(r)
if err != nil {
logs.Warn(err)
return
}
reader := bytes.NewReader(data)
p = ReadPom(reader)
reader.Seek(0, io.SeekStart)
cache.Save(path, reader)
}, repos...)
return p
}
func RegisterMavenOrigin(origin func(groupId, artifactId, version string) *Pom) {
if origin != nil {
mavenOrigin = func(groupId, artifactId, version string, repos ...common.RepoConfig) *Pom {
return origin(groupId, artifactId, version)
}
}
}
func DownloadPomFromRepo(dep PomDependency, do func(r io.Reader), repos ...common.RepoConfig) {
if !dep.Check() {
return
}
pom := fmt.Sprintf("%s/%s/%s/%s-%s.pom", strings.ReplaceAll(dep.GroupId, ".", "/"), dep.ArtifactId, dep.Version, dep.ArtifactId, dep.Version)
common.DownloadUrlFromRepos(pom, func(repo common.RepoConfig, r io.Reader) { do(r) }, append(defaultMavenRepo, repos...)...)
if !strings.HasSuffix(strings.ToLower(dep.Version), "-snapshot") {
return
}
snap := fmt.Sprintf("%s/%s/%s/maven-metadata.xml", strings.ReplaceAll(dep.GroupId, ".", "/"), dep.ArtifactId, dep.Version)
common.DownloadUrlFromRepos(snap, func(repo common.RepoConfig, r io.Reader) {
metadata := struct {
LastTime string `xml:"versioning>lastUpdated"`
SnapVersions []struct {
Version string `xml:"value"`
Time string `xml:"updated"`
} `xml:"versioning>snapshotVersions>snapshotVersion"`
}{}
err := xml.NewDecoder(r).Decode(&metadata)
if err != nil {
logs.Warn(err)
}
if metadata.LastTime == "" {
return
}
for _, snap := range metadata.SnapVersions {
if snap.Time == metadata.LastTime {
snapom := fmt.Sprintf("%s/%s/%s/%s-%s.pom", strings.ReplaceAll(dep.GroupId, ".", "/"), dep.ArtifactId, snap.Version, dep.ArtifactId, snap.Version)
common.DownloadUrlFromRepos(snapom, func(repo common.RepoConfig, r io.Reader) { do(r) }, repo)
break
}
}
}, append(defaultMavenRepo, repos...)...)
}
func MvnTree(ctx context.Context, pom *Pom) *model.DepGraph {
if !config.Conf().Optional.Dynamic {
return nil
}
if pom == nil {
return nil
}
if _, err := exec.LookPath("mvn"); err != nil {
return nil
}
cmd := exec.CommandContext(ctx, "mvn", "dependency:tree")
cmd.Dir = filepath.Dir(pom.File.Abspath())
output, err := cmd.CombinedOutput()
if err != nil {
return nil
}
var lines []string
tree := false
title := regexp.MustCompile(`--- [^\n]+ ---`)
scan := bufio.NewScanner(bytes.NewBuffer(output))
for scan.Scan() {
line := strings.TrimPrefix(scan.Text(), "[INFO] ")
if title.MatchString(line) {
tree = true
continue
}
if tree && strings.Trim(line, "-") == "" {
tree = false
root := parseMvnTree(lines)
if root != nil && root.Name == pom.ArtifactId {
root.Path = pom.File.Relpath()
return root
}
lines = nil
continue
}
if tree {
lines = append(lines, line)
continue
}
}
return nil
}
func parseMvnTree(lines []string) *model.DepGraph {
var tops = []*model.DepGraph{}
lastLevel := -1
_dep := model.NewDepGraphMap(nil, func(s ...string) *model.DepGraph {
return &model.DepGraph{
Vendor: s[0],
Name: s[1],
Version: s[2],
}
}).LoadOrStore
for _, line := range lines {
level := 0
for level*3+2 < len(line) && line[level*3+2] == ' ' {
level++
}
if level*3+2 >= len(line) {
continue
}
if level-lastLevel > 1 {
continue
}
tags := strings.Split(line[level*3:], ":")
if len(tags) < 4 {
continue
}
dep := _dep(tags[0], tags[1], tags[3])
if dep == nil {
continue
}
scope := tags[len(tags)-1]
if scope == "test" || scope == "provided" {
dep.Develop = true
}
if level > 0 {
tops[level-1].AppendChild(dep)
}
tops = append(tops[:level], dep)
lastLevel = level
}
if len(tops) > 0 {
return tops[0]
} else {
return nil
}
}