tools/compliance/readgraph.go - platform/build - Git at Google

 // Copyright 2021 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package compliance

 import (
 	"fmt"
 	"io"
 	"io/fs"
 	"os"
 	"strings"
 	"sync"

 	"android/soong/compliance/license_metadata_proto"

 	"google.golang.org/protobuf/encoding/prototext"
 )

 var (
 	// ConcurrentReaders is the size of the task pool for limiting resource usage e.g. open files.
 	ConcurrentReaders = 5
 )

 type globalFS struct{}

 var _ fs.FS = globalFS{}
 var _ fs.StatFS = globalFS{}

 func (s globalFS) Open(name string) (fs.File, error) {
 	return os.Open(name)
 }

 func (s globalFS) Stat(name string) (fs.FileInfo, error) {
 	return os.Stat(name)
 }

 var FS globalFS

 // GetFS returns a filesystem for accessing files under the OUT_DIR environment variable.
 func GetFS(outDir string) fs.FS {
 	if len(outDir) > 0 {
 		return os.DirFS(outDir)
 	}
 	return os.DirFS(".")
 }

 // result describes the outcome of reading and parsing a single license metadata file.
 type result struct {
 	// file identifies the path to the license metadata file
 	file string

 	// target contains the parsed metadata or nil if an error
 	target *TargetNode

 	// err is nil unless an error occurs
 	err error
 }

 // receiver coordinates the tasks for reading and parsing license metadata files.
 type receiver struct {
 	// lg accumulates the read metadata and becomes the final resulting LicenseGraph.
 	lg *LicenseGraph

 	// rootFS locates the root of the file system from which to read the files.
 	rootFS fs.FS

 	// stderr identifies the error output writer.
 	stderr io.Writer

 	// task provides a fixed-size task pool to limit concurrent open files etc.
 	task chan bool

 	// results returns one license metadata file result at a time.
 	results chan *result

 	// wg detects when done
 	wg sync.WaitGroup
 }

 // ReadLicenseGraph reads and parses `files` and their dependencies into a LicenseGraph.
 //
 // `files` become the root files of the graph for top-down walks of the graph.
 func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseGraph, error) {
 	if len(files) == 0 {
 		return nil, fmt.Errorf("no license metadata to analyze")
 	}
 	if ConcurrentReaders < 1 {
 		return nil, fmt.Errorf("need at least one task in pool")
 	}

 	lg := newLicenseGraph()
 	for _, f := range files {
 		if strings.HasSuffix(f, "meta_lic") {
 			lg.rootFiles = append(lg.rootFiles, f)
 		} else {
 			lg.rootFiles = append(lg.rootFiles, f+".meta_lic")
 		}
 	}

 	recv := &receiver{
 		lg:      lg,
 		rootFS:  rootFS,
 		stderr:  stderr,
 		task:    make(chan bool, ConcurrentReaders),
 		results: make(chan *result, ConcurrentReaders),
 		wg:      sync.WaitGroup{},
 	}
 	for i := 0; i < ConcurrentReaders; i++ {
 		recv.task <- true
 	}

 	readFiles := func() {
 		lg.mu.Lock()
 		// identify the metadata files to schedule reading tasks for
 		for _, f := range lg.rootFiles {
 			lg.targets[f] = nil
 		}
 		lg.mu.Unlock()

 		// schedule tasks to read the files
 		for _, f := range lg.rootFiles {
 			readFile(recv, f)
 		}

 		// schedule a task to wait until finished and close the channel.
 		go func() {
 			recv.wg.Wait()
 			close(recv.task)
 			close(recv.results)
 		}()
 	}
 	go readFiles()

 	// tasks to read license metadata files are scheduled; read and process results from channel
 	var err error
 	for recv.results != nil {
 		select {
 		case r, ok := <-recv.results:
 			if ok {
 				// handle errors by nil'ing ls, setting err, and clobbering results channel
 				if r.err != nil {
 					err = r.err
 					fmt.Fprintf(recv.stderr, "%s\n", err.Error())
 					lg = nil
 					recv.results = nil
 					continue
 				}

 				// record the parsed metadata (guarded by mutex)
 				recv.lg.mu.Lock()
 				lg.targets[r.target.name] = r.target
 				recv.lg.mu.Unlock()
 			} else {
 				// finished -- nil the results channel
 				recv.results = nil
 			}
 		}
 	}

 	if lg != nil {
 		esize := 0
 		for _, tn := range lg.targets {
 			esize += len(tn.proto.Deps)
 		}
 		lg.edges = make(TargetEdgeList, 0, esize)
 		for _, tn := range lg.targets {
 			tn.licenseConditions = LicenseConditionSetFromNames(tn.proto.LicenseConditions...)
 			err = addDependencies(lg, tn)
 			if err != nil {
 				return nil, fmt.Errorf("error indexing dependencies for %q: %w", tn.name, err)
 			}
 			tn.proto.Deps = []*license_metadata_proto.AnnotatedDependency{}
 		}
 	}
 	return lg, err

 }

 // targetNode contains the license metadata for a node in the license graph.
 type targetNode struct {
 	proto license_metadata_proto.LicenseMetadata

 	// name is the path to the metadata file.
 	name string

 	// lg is the license graph the node belongs to.
 	lg *LicenseGraph

 	// edges identifies the dependencies of the target.
 	edges TargetEdgeList

 	// licenseConditions identifies the set of license conditions originating at the target node.
 	licenseConditions LicenseConditionSet

 	// resolution identifies the set of conditions resolved by acting on the target node.
 	resolution LicenseConditionSet

 	// pure indicates whether to treat the node as a pure aggregate (no internal linkage)
 	pure bool
 }

 // addDependencies converts the proto AnnotatedDependencies into `edges`
 func addDependencies(lg *LicenseGraph, tn *TargetNode) error {
 	tn.edges = make(TargetEdgeList, 0, len(tn.proto.Deps))
 	for _, ad := range tn.proto.Deps {
 		dependency := ad.GetFile()
 		if len(dependency) == 0 {
 			return fmt.Errorf("missing dependency name")
 		}
 		dtn, ok := lg.targets[dependency]
 		if !ok {
 			return fmt.Errorf("unknown dependency name %q", dependency)
 		}
 		if dtn == nil {
 			return fmt.Errorf("nil dependency for name %q", dependency)
 		}
 		annotations := newEdgeAnnotations()
 		for _, a := range ad.Annotations {
 			// look up a common constant annotation string from a small map
 			// instead of creating 1000's of copies of the same 3 strings.
 			if ann, ok := RecognizedAnnotations[a]; ok {
 				annotations.annotations[ann] = struct{}{}
 			}
 		}
 		edge := &TargetEdge{tn, dtn, annotations}
 		lg.edges = append(lg.edges, edge)
 		tn.edges = append(tn.edges, edge)
 	}
 	return nil
 }

 // readFile is a task to read and parse a single license metadata file, and to schedule
 // additional tasks for reading and parsing dependencies as necessary.
 func readFile(recv *receiver, file string) {
 	recv.wg.Add(1)
 	<-recv.task
 	go func() {
 		f, err := recv.rootFS.Open(file)
 		if err != nil {
 			recv.results <- &result{file, nil, fmt.Errorf("error opening license metadata %q: %w", file, err)}
 			return
 		}

 		// read the file
 		data, err := io.ReadAll(f)
 		if err != nil {
 			recv.results <- &result{file, nil, fmt.Errorf("error reading license metadata %q: %w", file, err)}
 			return
 		}
 		f.Close()

 		tn := &TargetNode{lg: recv.lg, name: file}

 		err = prototext.Unmarshal(data, &tn.proto)
 		if err != nil {
 			recv.results <- &result{file, nil, fmt.Errorf("error license metadata %q: %w", file, err)}
 			return
 		}

 		// send result for this file and release task before scheduling dependencies,
 		// but do not signal done to WaitGroup until dependencies are scheduled.
 		recv.results <- &result{file, tn, nil}
 		recv.task <- true

 		// schedule tasks as necessary to read dependencies
 		for _, ad := range tn.proto.Deps {
 			dependency := ad.GetFile()
 			// decide, signal and record whether to schedule task in critical section
 			recv.lg.mu.Lock()
 			_, alreadyScheduled := recv.lg.targets[dependency]
 			if !alreadyScheduled {
 				recv.lg.targets[dependency] = nil
 			}
 			recv.lg.mu.Unlock()
 			// schedule task to read dependency file outside critical section
 			if !alreadyScheduled {
 				readFile(recv, dependency)
 			}
 		}

 		// signal task done after scheduling dependencies
 		recv.wg.Done()
 	}()
 }
	// Copyright 2021 Google LLC
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package compliance

	import (
	"fmt"
	"io"
	"io/fs"
	"os"
	"strings"
	"sync"

	"android/soong/compliance/license_metadata_proto"

	"google.golang.org/protobuf/encoding/prototext"
	)

	var (
	// ConcurrentReaders is the size of the task pool for limiting resource usage e.g. open files.
	ConcurrentReaders = 5
	)

	type globalFS struct{}

	var _ fs.FS = globalFS{}
	var _ fs.StatFS = globalFS{}

	func (s globalFS) Open(name string) (fs.File, error) {
	return os.Open(name)
	}

	func (s globalFS) Stat(name string) (fs.FileInfo, error) {
	return os.Stat(name)
	}

	var FS globalFS

	// GetFS returns a filesystem for accessing files under the OUT_DIR environment variable.
	func GetFS(outDir string) fs.FS {
	if len(outDir) > 0 {
	return os.DirFS(outDir)
	}
	return os.DirFS(".")
	}

	// result describes the outcome of reading and parsing a single license metadata file.
	type result struct {
	// file identifies the path to the license metadata file
	file string

	// target contains the parsed metadata or nil if an error
	target *TargetNode

	// err is nil unless an error occurs
	err error
	}

	// receiver coordinates the tasks for reading and parsing license metadata files.
	type receiver struct {
	// lg accumulates the read metadata and becomes the final resulting LicenseGraph.
	lg *LicenseGraph

	// rootFS locates the root of the file system from which to read the files.
	rootFS fs.FS

	// stderr identifies the error output writer.
	stderr io.Writer

	// task provides a fixed-size task pool to limit concurrent open files etc.
	task chan bool

	// results returns one license metadata file result at a time.
	results chan *result

	// wg detects when done
	wg sync.WaitGroup
	}

	// ReadLicenseGraph reads and parses `files` and their dependencies into a LicenseGraph.
	//
	// `files` become the root files of the graph for top-down walks of the graph.
	func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseGraph, error) {
	if len(files) == 0 {
	return nil, fmt.Errorf("no license metadata to analyze")
	}
	if ConcurrentReaders < 1 {
	return nil, fmt.Errorf("need at least one task in pool")
	}

	lg := newLicenseGraph()
	for _, f := range files {
	if strings.HasSuffix(f, "meta_lic") {
	lg.rootFiles = append(lg.rootFiles, f)
	} else {
	lg.rootFiles = append(lg.rootFiles, f+".meta_lic")
	}
	}

	recv := &receiver{
	lg: lg,
	rootFS: rootFS,
	stderr: stderr,
	task: make(chan bool, ConcurrentReaders),
	results: make(chan *result, ConcurrentReaders),
	wg: sync.WaitGroup{},
	}
	for i := 0; i < ConcurrentReaders; i++ {
	recv.task <- true
	}

	readFiles := func() {
	lg.mu.Lock()
	// identify the metadata files to schedule reading tasks for
	for _, f := range lg.rootFiles {
	lg.targets[f] = nil
	}
	lg.mu.Unlock()

	// schedule tasks to read the files
	for _, f := range lg.rootFiles {
	readFile(recv, f)
	}

	// schedule a task to wait until finished and close the channel.
	go func() {
	recv.wg.Wait()
	close(recv.task)
	close(recv.results)
	}()
	}
	go readFiles()

	// tasks to read license metadata files are scheduled; read and process results from channel
	var err error
	for recv.results != nil {
	select {
	case r, ok := <-recv.results:
	if ok {
	// handle errors by nil'ing ls, setting err, and clobbering results channel
	if r.err != nil {
	err = r.err
	fmt.Fprintf(recv.stderr, "%s\n", err.Error())
	lg = nil
	recv.results = nil
	continue
	}

	// record the parsed metadata (guarded by mutex)
	recv.lg.mu.Lock()
	lg.targets[r.target.name] = r.target
	recv.lg.mu.Unlock()
	} else {
	// finished -- nil the results channel
	recv.results = nil
	}
	}
	}

	if lg != nil {
	esize := 0
	for _, tn := range lg.targets {
	esize += len(tn.proto.Deps)
	}
	lg.edges = make(TargetEdgeList, 0, esize)
	for _, tn := range lg.targets {
	tn.licenseConditions = LicenseConditionSetFromNames(tn.proto.LicenseConditions...)
	err = addDependencies(lg, tn)
	if err != nil {
	return nil, fmt.Errorf("error indexing dependencies for %q: %w", tn.name, err)
	}
	tn.proto.Deps = []*license_metadata_proto.AnnotatedDependency{}
	}
	}
	return lg, err

	}

	// targetNode contains the license metadata for a node in the license graph.
	type targetNode struct {
	proto license_metadata_proto.LicenseMetadata

	// name is the path to the metadata file.
	name string

	// lg is the license graph the node belongs to.
	lg *LicenseGraph

	// edges identifies the dependencies of the target.
	edges TargetEdgeList

	// licenseConditions identifies the set of license conditions originating at the target node.
	licenseConditions LicenseConditionSet

	// resolution identifies the set of conditions resolved by acting on the target node.
	resolution LicenseConditionSet

	// pure indicates whether to treat the node as a pure aggregate (no internal linkage)
	pure bool
	}

	// addDependencies converts the proto AnnotatedDependencies into `edges`
	func addDependencies(lg LicenseGraph, tn TargetNode) error {
	tn.edges = make(TargetEdgeList, 0, len(tn.proto.Deps))
	for _, ad := range tn.proto.Deps {
	dependency := ad.GetFile()
	if len(dependency) == 0 {
	return fmt.Errorf("missing dependency name")
	}
	dtn, ok := lg.targets[dependency]
	if !ok {
	return fmt.Errorf("unknown dependency name %q", dependency)
	}
	if dtn == nil {
	return fmt.Errorf("nil dependency for name %q", dependency)
	}
	annotations := newEdgeAnnotations()
	for _, a := range ad.Annotations {
	// look up a common constant annotation string from a small map
	// instead of creating 1000's of copies of the same 3 strings.
	if ann, ok := RecognizedAnnotations[a]; ok {
	annotations.annotations[ann] = struct{}{}
	}
	}
	edge := &TargetEdge{tn, dtn, annotations}
	lg.edges = append(lg.edges, edge)
	tn.edges = append(tn.edges, edge)
	}
	return nil
	}

	// readFile is a task to read and parse a single license metadata file, and to schedule
	// additional tasks for reading and parsing dependencies as necessary.
	func readFile(recv *receiver, file string) {
	recv.wg.Add(1)
	<-recv.task
	go func() {
	f, err := recv.rootFS.Open(file)
	if err != nil {
	recv.results <- &result{file, nil, fmt.Errorf("error opening license metadata %q: %w", file, err)}
	return
	}

	// read the file
	data, err := io.ReadAll(f)
	if err != nil {
	recv.results <- &result{file, nil, fmt.Errorf("error reading license metadata %q: %w", file, err)}
	return
	}
	f.Close()

	tn := &TargetNode{lg: recv.lg, name: file}

	err = prototext.Unmarshal(data, &tn.proto)
	if err != nil {
	recv.results <- &result{file, nil, fmt.Errorf("error license metadata %q: %w", file, err)}
	return
	}

	// send result for this file and release task before scheduling dependencies,
	// but do not signal done to WaitGroup until dependencies are scheduled.
	recv.results <- &result{file, tn, nil}
	recv.task <- true

	// schedule tasks as necessary to read dependencies
	for _, ad := range tn.proto.Deps {
	dependency := ad.GetFile()
	// decide, signal and record whether to schedule task in critical section
	recv.lg.mu.Lock()
	_, alreadyScheduled := recv.lg.targets[dependency]
	if !alreadyScheduled {
	recv.lg.targets[dependency] = nil
	}
	recv.lg.mu.Unlock()
	// schedule task to read dependency file outside critical section
	if !alreadyScheduled {
	readFile(recv, dependency)
	}
	}

	// signal task done after scheduling dependencies
	recv.wg.Done()
	}()
	}