Merge "A update: Add rest of make push aliases" into main
diff --git a/OWNERS_ADTE_TEAM b/OWNERS_ADTE_TEAM
index f4fee56..76e26bc 100644
--- a/OWNERS_ADTE_TEAM
+++ b/OWNERS_ADTE_TEAM
@@ -1,5 +1,4 @@
# ADTE team
-agueeva@google.com
davidjames@google.com
hwj@google.com
hzalek@google.com
diff --git a/adevice/src/metrics.rs b/adevice/src/metrics.rs
index 2bc3722..dd614fd 100644
--- a/adevice/src/metrics.rs
+++ b/adevice/src/metrics.rs
@@ -14,14 +14,14 @@
use std::fs;
use std::process::{Command, Stdio};
use std::time::UNIX_EPOCH;
-use tracing::debug;
+use tracing::info;
use uuid::Uuid;
const ENV_OUT: &str = "OUT";
const ENV_USER: &str = "USER";
const ENV_TARGET: &str = "TARGET_PRODUCT";
const ENV_SURVEY_BANNER: &str = "ADEVICE_SURVEY_BANNER";
-const METRICS_UPLOADER: &str = "/google/bin/releases/adevice-dev/";
+const METRICS_UPLOADER: &str = "/google/bin/releases/adevice-dev/metrics_uploader";
const ADEVICE_LOG_SOURCE: i32 = 2265;
pub trait MetricSender {
@@ -219,7 +219,7 @@
fn drop(&mut self) {
match self.send() {
Ok(_) => (),
- Err(e) => debug!("Failed to send metrics: {}", e),
+ Err(e) => info!("Failed to send metrics: {}", e),
};
}
}
diff --git a/atest/atest_execution_info.py b/atest/atest_execution_info.py
index dd4188b..0f35b98 100644
--- a/atest/atest_execution_info.py
+++ b/atest/atest_execution_info.py
@@ -349,6 +349,24 @@
self._proc_usb_speed.terminate()
log_path = pathlib.Path(self.work_dir)
+
+ build_log_path = log_path / 'build_logs'
+ build_log_path.mkdir()
+ AtestExecutionInfo._copy_build_artifacts_to_log_dir(
+ self._start_time,
+ time.time(),
+ self._repo_out_dir,
+ build_log_path,
+ 'build.trace',
+ )
+ AtestExecutionInfo._copy_build_artifacts_to_log_dir(
+ self._start_time,
+ time.time(),
+ self._repo_out_dir,
+ build_log_path,
+ 'verbose.log',
+ )
+
html_path = None
if self.result_file_obj and not has_non_test_options(self.args_ns):
@@ -386,9 +404,6 @@
logging.debug('handle_exc_and_send_exit_event:%s', main_exit_code)
metrics_utils.handle_exc_and_send_exit_event(main_exit_code)
- AtestExecutionInfo._copy_build_trace_to_log_dir(
- self._start_time, time.time(), self._repo_out_dir, log_path
- )
if log_uploader.is_uploading_logs():
log_uploader.upload_logs_detached(log_path)
@@ -430,20 +445,29 @@
return f'http://go/from-atest-runid/{metrics.get_run_id()}'
@staticmethod
- def _copy_build_trace_to_log_dir(
+ def _copy_build_artifacts_to_log_dir(
start_time: float,
end_time: float,
repo_out_path: pathlib.Path,
log_path: pathlib.Path,
+ file_name_prefix: str,
):
+ """Copy build trace files to log directory.
+ Params:
+ start_time: The start time of the build.
+ end_time: The end time of the build.
+ repo_out_path: The path to the repo out directory.
+ log_path: The path to the log directory.
+ file_name_prefix: The prefix of the file name.
+ """
for file in repo_out_path.iterdir():
if (
file.is_file()
- and file.name.startswith('build.trace')
+ and file.name.startswith(file_name_prefix)
and start_time <= file.stat().st_mtime <= end_time
):
- shutil.copy(file, log_path)
+ shutil.copy(file, log_path / file.name)
@staticmethod
def _generate_execution_detail(args):
diff --git a/atest/atest_execution_info_unittest.py b/atest/atest_execution_info_unittest.py
index 3c176f3..af61454 100755
--- a/atest/atest_execution_info_unittest.py
+++ b/atest/atest_execution_info_unittest.py
@@ -52,7 +52,7 @@
self.setUpPyfakefs()
self.fs.create_dir(constants.ATEST_RESULT_ROOT)
- def test_copy_build_trace_to_log_dir_new_trace_copy(self):
+ def test_copy_build_artifacts_to_log_dir_new_trace_copy(self):
start_time = 10
log_path = pathlib.Path('/logs')
self.fs.create_dir(log_path)
@@ -63,15 +63,15 @@
os.utime(build_trace_path, (20, 20))
end_time = 30
- aei.AtestExecutionInfo._copy_build_trace_to_log_dir(
- start_time, end_time, out_path, log_path
+ aei.AtestExecutionInfo._copy_build_artifacts_to_log_dir(
+ start_time, end_time, out_path, log_path, 'build.trace'
)
self.assertTrue(
self._is_dir_contains_files_with_prefix(log_path, 'build.trace')
)
- def test_copy_build_trace_to_log_dir_old_trace_does_not_copy(self):
+ def test_copy_build_artifacts_to_log_dir_old_trace_does_not_copy(self):
start_time = 10
log_path = pathlib.Path('/logs')
self.fs.create_dir(log_path)
@@ -82,8 +82,8 @@
os.utime(build_trace_path, (5, 5))
end_time = 30
- aei.AtestExecutionInfo._copy_build_trace_to_log_dir(
- start_time, end_time, out_path, log_path
+ aei.AtestExecutionInfo._copy_build_artifacts_to_log_dir(
+ start_time, end_time, out_path, log_path, 'build.trace'
)
self.assertFalse(
@@ -95,8 +95,8 @@
log_path = pathlib.Path('/logs')
self.fs.create_dir(log_path)
out_path = pathlib.Path('/out')
- build_trace_path1 = out_path / 'build.trace.1'
- build_trace_path2 = out_path / 'build.trace.2'
+ build_trace_path1 = out_path / 'build.trace.1.gz'
+ build_trace_path2 = out_path / 'build.trace.2.gz'
self.fs.create_file(build_trace_path1)
self.fs.create_file(build_trace_path2)
# Set the trace file's mtime greater than start time
@@ -104,15 +104,15 @@
os.utime(build_trace_path2, (20, 20))
end_time = 30
- aei.AtestExecutionInfo._copy_build_trace_to_log_dir(
- start_time, end_time, out_path, log_path
+ aei.AtestExecutionInfo._copy_build_artifacts_to_log_dir(
+ start_time, end_time, out_path, log_path, 'build.trace'
)
self.assertTrue(
- self._is_dir_contains_files_with_prefix(log_path, 'build.trace.1')
+ self._is_dir_contains_files_with_prefix(log_path, 'build.trace.1.gz')
)
self.assertTrue(
- self._is_dir_contains_files_with_prefix(log_path, 'build.trace.2')
+ self._is_dir_contains_files_with_prefix(log_path, 'build.trace.2.gz')
)
def _is_dir_contains_files_with_prefix(
diff --git a/atest/atest_utils.py b/atest/atest_utils.py
index 485a102..a893d60 100644
--- a/atest/atest_utils.py
+++ b/atest/atest_utils.py
@@ -20,6 +20,7 @@
from __future__ import print_function
+from collections import deque
from dataclasses import dataclass
import datetime
import enum
@@ -41,7 +42,7 @@
import sys
from threading import Thread
import traceback
-from typing import Any, Dict, List, Set, Tuple
+from typing import Any, Dict, IO, List, Set, Tuple
import urllib
import xml.etree.ElementTree as ET
import zipfile
@@ -53,7 +54,9 @@
from atest.metrics import metrics_utils
from atest.tf_proto import test_record_pb2
-_BASH_RESET_CODE = '\033[0m\n'
+_BUILD_OUTPUT_ROLLING_LINES = 6
+_BASH_CLEAR_PREVIOUS_LINE_CODE = '\033[F\033[K'
+_BASH_RESET_CODE = '\033[0m'
DIST_OUT_DIR = Path(
os.environ.get(constants.ANDROID_BUILD_TOP, os.getcwd()) + '/out/dist/'
)
@@ -267,13 +270,57 @@
return output
-# TODO: b/187122993 refine subprocess with 'with-statement' in fixit week.
-def run_limited_output(cmd, env_vars=None):
+def _stream_io_output(io_input: IO, io_output: IO, max_lines=None):
+ """Stream an IO output with max number of rolling lines to display if set.
+
+ Args:
+ input: The file-like object to read the output from.
+ output: The file-like object to write the output to.
+ max_lines: The maximum number of rolling lines to display. If None, all
+ lines will be displayed.
+ """
+ print('\n----------------------------------------------------')
+ term_width, _ = get_terminal_size()
+ full_output = []
+ last_lines = None if not max_lines else deque(maxlen=max_lines)
+ last_number_of_lines = 0
+ for line in iter(io_input.readline, ''):
+ full_output.append(line)
+ line = line.rstrip()
+ if last_lines is None:
+ io_output.write(line)
+ io_output.write('\n')
+ io_output.flush()
+ continue
+ # Split the line if it's longer than the terminal width
+ wrapped_lines = (
+ [line]
+ if len(line) <= term_width
+ else [line[i : i + term_width] for i in range(0, len(line), term_width)]
+ )
+ last_lines.extend(wrapped_lines)
+ io_output.write(_BASH_CLEAR_PREVIOUS_LINE_CODE * last_number_of_lines)
+ io_output.write('\n'.join(last_lines))
+ io_output.write('\n')
+ io_output.flush()
+ last_number_of_lines = len(last_lines)
+ io_input.close()
+ io_output.write(_BASH_RESET_CODE)
+ io_output.flush()
+ print('----------------------------------------------------')
+
+
+def run_limited_output(
+ cmd, env_vars=None, shell=False, start_new_session=False
+):
"""Runs a given command and streams the output on a single line in stdout.
Args:
cmd: A list of strings representing the command to run.
env_vars: Optional arg. Dict of env vars to set during build.
+ shell: Optional arg. Whether to use shell to run the command.
+ start_new_session: Optional arg. Whether to start a new session for the
+ command.
Raises:
subprocess.CalledProcessError: When the command exits with a non-0
@@ -281,35 +328,20 @@
"""
# Send stderr to stdout so we only have to deal with a single pipe.
with subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env_vars
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=env_vars,
+ shell=shell,
+ start_new_session=start_new_session,
+ text=True,
) as proc:
- sys.stdout.write('\n')
- term_width, _ = get_terminal_size()
- white_space = ' ' * int(term_width)
- full_output = []
- while proc.poll() is None:
- line = proc.stdout.readline().decode('utf-8')
- # Readline will often return empty strings.
- if not line:
- continue
- full_output.append(line)
- # Trim the line to the width of the terminal.
- # Note: Does not handle terminal resizing, which is probably not
- # worth checking the width every loop.
- if len(line) >= term_width:
- line = line[: term_width - 1]
- # Clear the last line we outputted.
- sys.stdout.write('\r%s\r' % white_space)
- sys.stdout.write('%s' % line.strip())
- sys.stdout.flush()
- # Reset stdout (on bash) to remove any custom formatting and newline.
- sys.stdout.write(_BASH_RESET_CODE)
- sys.stdout.flush()
- # Wait for the Popen to finish completely before checking the
- # returncode.
- proc.wait()
- if proc.returncode != 0:
- raise subprocess.CalledProcessError(proc.returncode, cmd, full_output)
+ _stream_io_output(
+ proc.stdout, _original_sys_stdout, _BUILD_OUTPUT_ROLLING_LINES
+ )
+ returncode = proc.wait()
+ if returncode:
+ raise subprocess.CalledProcessError(returncode, cmd, full_output)
def get_build_out_dir(*joinpaths) -> Path:
diff --git a/atest/atest_utils_unittest.py b/atest/atest_utils_unittest.py
index 75bc523..0b8d6a3 100755
--- a/atest/atest_utils_unittest.py
+++ b/atest/atest_utils_unittest.py
@@ -66,6 +66,62 @@
----------------------------
"""
+class StreamIoOutputTest(unittest.TestCase):
+ """Class that tests the _stream_io_output function."""
+
+ def test_stream_io_output_no_max_lines_no_clear_line_code(self):
+ """Test when max_lines is None, no clear line code is written to the stream."""
+ io_input = StringIO()
+ io_input.write(f'1\n' * 10)
+ io_input.seek(0)
+ io_output = StringIO()
+
+ atest_utils._stream_io_output(io_input, io_output, max_lines=None)
+
+ self.assertNotIn(atest_utils._BASH_CLEAR_PREVIOUS_LINE_CODE, io_output.getvalue())
+
+ @mock.patch.object(atest_utils, 'get_terminal_size', return_value=(5, -1))
+ def test_stream_io_output_wrap_long_lines(self, _):
+ """Test when max_lines is set, long lines will be wrapped."""
+ io_input = StringIO()
+ io_input.write(f'1' * 10)
+ io_input.seek(0)
+ io_output = StringIO()
+
+ atest_utils._stream_io_output(io_input, io_output, max_lines=10)
+
+ self.assertIn('11111\n11111', io_output.getvalue())
+
+ @mock.patch.object(atest_utils, 'get_terminal_size', return_value=(5, -1))
+ def test_stream_io_output_clear_lines_over_max_lines(self, _):
+ """Test when line exceeds max_lines, the previous lines are cleared."""
+ io_input = StringIO()
+ io_input.write('1\n2\n3\n')
+ io_input.seek(0)
+ io_output = StringIO()
+
+ atest_utils._stream_io_output(io_input, io_output, max_lines=2)
+
+ self.assertIn(
+ atest_utils._BASH_CLEAR_PREVIOUS_LINE_CODE * 2 + '2\n3\n',
+ io_output.getvalue(),
+ )
+
+ @mock.patch.object(atest_utils, 'get_terminal_size', return_value=(5, -1))
+ def test_stream_io_output_no_clear_lines_under_max_lines(self, _):
+ """Test when line is under max_lines, the previous lines are not cleared."""
+ io_input = StringIO()
+ io_input.write('1\n2\n3\n')
+ io_input.seek(0)
+ io_output = StringIO()
+
+ atest_utils._stream_io_output(io_input, io_output, max_lines=4)
+
+ self.assertIn(
+ atest_utils._BASH_CLEAR_PREVIOUS_LINE_CODE * 2 + '1\n2\n3\n',
+ io_output.getvalue(),
+ )
+
class ConcatenatePathTest(unittest.TestCase):
"""Class that tests path concatenation."""
diff --git a/atest/coverage/coverage.py b/atest/coverage/coverage.py
index b99cbb4..b1c49f0 100644
--- a/atest/coverage/coverage.py
+++ b/atest/coverage/coverage.py
@@ -249,7 +249,7 @@
continue
module_dir = soong_intermediates.joinpath(path, module)
# Check for unstripped binaries to report coverage.
- report_binaries.update(_find_native_binaries(module_dir))
+ report_binaries.update(module_dir.glob('*cov*/**/unstripped/*'))
# Host tests use the test itself to generate the coverage report.
info = mod_info.get_module_info(module)
@@ -264,26 +264,21 @@
str(f) for f in mod_info.get_installed_paths(module)
)
- return report_binaries
+ return _strip_irrelevant_objects(report_binaries)
-def _find_native_binaries(module_dir):
- files = module_dir.glob('*cov*/**/unstripped/*')
-
- # Exclude .rsp files. These are files containing the command line used to
- # generate the unstripped binaries, but are stored in the same directory as
- # the actual output binary.
- # Exclude .d and .d.raw files. These are Rust dependency files and are also
- # stored in the unstripped directory.
- # Exclude .toc files. These are just a table of conents of a shared library,
- # but are also stored in the unstripped directory.
- return [
- str(file)
- for file in files
- if '.rsp' not in file.suffixes
- and '.d' not in file.suffixes
- and '.toc' not in file.suffixes
- ]
+def _strip_irrelevant_objects(files):
+ objects = set()
+ for file in files:
+ cmd = ['llvm-readobj', file]
+ try:
+ subprocess.run(
+ cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+ )
+ objects.add(file)
+ except subprocess.CalledProcessError:
+ logging.debug(f'{file} is not a valid object file, skipping.')
+ return objects
def _get_all_src_paths(modules, mod_info):
diff --git a/atest/coverage/coverage_unittest.py b/atest/coverage/coverage_unittest.py
index 36194ae..690f997 100755
--- a/atest/coverage/coverage_unittest.py
+++ b/atest/coverage/coverage_unittest.py
@@ -149,7 +149,14 @@
return_value=PosixPath('/out/soong/.intermediates'),
)
@mock.patch.object(PosixPath, 'glob')
- def test_native_binary(self, _glob, _get_build_out_dir):
+ @mock.patch.object(
+ coverage,
+ '_strip_irrelevant_objects',
+ return_value={
+ '/out/soong/.intermediates/path/to/native_bin/variant-name-cov/unstripped/native_bin'
+ },
+ )
+ def test_native_binary(self, _strip_irrelevant_objects, _glob, _get_build_out_dir):
_glob.return_value = [
PosixPath(
'/out/soong/.intermediates/path/to/native_bin/variant-name-cov/unstripped/native_bin'
@@ -175,7 +182,14 @@
return_value=PosixPath('/out/soong/.intermediates'),
)
@mock.patch.object(PosixPath, 'glob')
- def test_skip_rsp_and_d_and_toc_files(self, _glob, _get_build_out_dir):
+ @mock.patch.object(
+ coverage,
+ '_strip_irrelevant_objects',
+ return_value={
+ '/out/soong/.intermediates/path/to/native_bin/variant-name-cov/unstripped/native_bin'
+ },
+ )
+ def test_skip_rsp_and_d_and_toc_files(self, _strip_irrelevant_objects, _glob, _get_build_out_dir):
_glob.return_value = [
PosixPath(
'/out/soong/.intermediates/path/to/native_bin/variant-name-cov/unstripped/native_bin'
@@ -204,7 +218,14 @@
},
)
- def test_host_test_includes_installed(self):
+ @mock.patch.object(
+ coverage,
+ '_strip_irrelevant_objects',
+ return_value={
+ '/out/host/nativetests/native_host_test'
+ },
+ )
+ def test_host_test_includes_installed(self, _strip_irrelevant_objects):
code_under_test = {'native_host_test'}
mod_info = create_module_info([
module(
diff --git a/atest/result_reporter.py b/atest/result_reporter.py
index 32a76e8..4e175d3 100644
--- a/atest/result_reporter.py
+++ b/atest/result_reporter.py
@@ -400,7 +400,12 @@
for runner_name, groups in self.runners.items():
for group_name, stats in groups.items():
name = group_name if group_name else runner_name
- summary = self.process_summary(name, stats)
+ test_run_name = (
+ self.all_test_results[-1].test_run_name
+ if self.all_test_results[-1].test_run_name != name
+ else None
+ )
+ summary = self.process_summary(name, stats, test_run_name=test_run_name)
run_summary.append(summary)
summary_list = ITER_SUMMARY.get(iteration_num, [])
summary_list.extend(run_summary)
@@ -616,7 +621,7 @@
for test_name in self.failed_tests:
print(test_name)
- def process_summary(self, name, stats):
+ def process_summary(self, name, stats, test_run_name=None):
"""Process the summary line.
Strategy:
@@ -632,6 +637,7 @@
Args:
name: A string of test name.
stats: A RunStat instance for a test group.
+ test_run_name: A string of test run name (optional)
Returns:
A summary of the test result.
@@ -687,8 +693,9 @@
)
ITER_COUNTS[name] = temp
+ summary_name = f'{name}:{test_run_name}' if test_run_name else name
summary = '%s: %s: %s, %s: %s, %s: %s, %s: %s %s %s' % (
- name,
+ summary_name,
passed_label,
stats.passed,
failed_label,
