Add fileslist.go to calculate hashes in parallel
Bug: 36274890
Test: Manual
Change-Id: I548da5607cf3b993ad21cbb04a57fcbd5bfb7f51
diff --git a/cmd/fileslist/Android.bp b/cmd/fileslist/Android.bp
new file mode 100644
index 0000000..cbf939a
--- /dev/null
+++ b/cmd/fileslist/Android.bp
@@ -0,0 +1,20 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+ name: "fileslist",
+ srcs: [
+ "fileslist.go",
+ ],
+}
diff --git a/cmd/fileslist/fileslist.go b/cmd/fileslist/fileslist.go
new file mode 100755
index 0000000..1cf948f
--- /dev/null
+++ b/cmd/fileslist/fileslist.go
@@ -0,0 +1,165 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// fileslist.py replacement written in GO, which utilizes multi-cores.
+
+package main
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+ "sync"
+)
+
+const (
+ MAX_DEFAULT_PARA = 24
+)
+
+func defaultPara() int {
+ ret := runtime.NumCPU()
+ if ret > MAX_DEFAULT_PARA {
+ return MAX_DEFAULT_PARA
+ }
+ return ret
+}
+
+var (
+ para = flag.Int("para", defaultPara(), "Number of goroutines")
+)
+
+// Represents each file.
+type Node struct {
+ SHA256 string
+ Name string // device side path.
+ Size int64
+ path string // host side path.
+ stat os.FileInfo
+}
+
+func newNode(hostPath string, devicePath string, stat os.FileInfo) Node {
+ return Node{Name: devicePath, path: hostPath, stat: stat}
+}
+
+// Scan a Node and returns true if it should be added to the result.
+func (n *Node) scan() bool {
+ n.Size = n.stat.Size()
+
+ // Calculate SHA256.
+ f, err := os.Open(n.path)
+ if err != nil {
+ // If the file can't be read, it's probably a symlink to an absolute path...
+ // Returns the following to mimic the behavior of fileslist.py.
+ n.SHA256 = "----------------------------------------------------------------"
+ return true
+ }
+ defer f.Close()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ panic(err)
+ }
+ n.SHA256 = fmt.Sprintf("%x", h.Sum(nil))
+ return true
+}
+
+func main() {
+ flag.Parse()
+
+ allOutput := make([]Node, 0, 1024) // Store all outputs.
+ mutex := &sync.Mutex{} // Guard allOutput
+
+ ch := make(chan Node) // Pass nodes to goroutines.
+
+ var wg sync.WaitGroup // To wait for all goroutines.
+ wg.Add(*para)
+
+ // Scan files in multiple goroutines.
+ for i := 0; i < *para; i++ {
+ go func() {
+ defer wg.Done()
+
+ output := make([]Node, 0, 1024) // Local output list.
+ for node := range ch {
+ if node.scan() {
+ output = append(output, node)
+ }
+ }
+ // Add to the global output list.
+ mutex.Lock()
+ allOutput = append(allOutput, output...)
+ mutex.Unlock()
+ }()
+ }
+
+ // Walk the directories and find files to scan.
+ for _, dir := range flag.Args() {
+ absDir, err := filepath.Abs(dir)
+ if err != nil {
+ panic(err)
+ }
+ deviceRoot := filepath.Clean(absDir + "/..")
+ err = filepath.Walk(dir, func(path string, stat os.FileInfo, err error) error {
+ if err != nil {
+ panic(err)
+ }
+ if stat.IsDir() {
+ return nil
+ }
+ absPath, err := filepath.Abs(path)
+ if err != nil {
+ panic(err)
+ }
+ devicePath, err := filepath.Rel(deviceRoot, absPath)
+ if err != nil {
+ panic(err)
+ }
+ devicePath = "/" + devicePath
+ ch <- newNode(absPath, devicePath, stat)
+ return nil
+ })
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ // Wait until all the goroutines finish.
+ close(ch)
+ wg.Wait()
+
+ // Sort the entries and dump as json.
+ sort.Slice(allOutput, func(i, j int) bool {
+ if allOutput[i].Size > allOutput[j].Size {
+ return true
+ }
+ if allOutput[i].Size == allOutput[j].Size && strings.Compare(allOutput[i].Name, allOutput[j].Name) > 0 {
+ return true
+ }
+ return false
+ })
+
+ j, err := json.MarshalIndent(allOutput, "", " ")
+ if err != nil {
+ panic(nil)
+ }
+
+ fmt.Printf("%s\n", j)
+}
