Save node info to BigQuery (#27113)
* save nodeinfo to bigquery
* correct format
diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py
index be447db..0cb538b 100755
--- a/tools/run_tests/performance/bq_upload_result.py
+++ b/tools/run_tests/performance/bq_upload_result.py
@@ -63,7 +63,7 @@
def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file,
- metadata_file):
+ metadata_file, node_info_file):
with open(result_file, 'r') as f:
scenario_result = json.loads(f.read())
@@ -71,7 +71,7 @@
_create_results_table(bq, dataset_id, table_id)
if not _insert_scenario_result(bq, dataset_id, table_id, scenario_result,
- metadata_file):
+ metadata_file, node_info_file):
print('Error uploading result to bigquery.')
sys.exit(1)
@@ -90,10 +90,12 @@
table_id,
scenario_result,
test_metadata_file,
+ node_info_file,
flatten=True):
if flatten:
_flatten_result_inplace(scenario_result)
_populate_metadata_from_file(scenario_result, test_metadata_file)
+ _populate_node_metadata_from_file(scenario_result, node_info_file)
row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result)
return big_query_utils.insert_rows(bq, _PROJECT_ID, dataset_id, table_id,
[row])
@@ -213,6 +215,30 @@
scenario_result['metadata'] = metadata
+def _populate_node_metadata_from_file(scenario_result, node_info_file):
+ node_metadata = {'driver': {}, 'servers': [], 'clients': []}
+ _node_info_to_bq_node_metadata_key_map = {
+ 'Name': 'name',
+ 'PodIP': 'podIP',
+ 'NodeName': 'nodeName',
+ }
+ with open(node_info_file, 'r') as f:
+ file_metadata = json.loads(f.read())
+ for key, value in _node_info_to_bq_node_metadata_key_map.items():
+ node_metadata['driver'][value] = file_metadata['Driver'][key]
+ for clientNodeInfo in file_metadata['Clients']:
+ node_metadata['clients'].append({
+ value: clientNodeInfo[key] for key, value in
+ _node_info_to_bq_node_metadata_key_map.items()
+ })
+ for serverNodeInfo in file_metadata['Servers']:
+ node_metadata['servers'].append({
+ value: serverNodeInfo[key] for key, value in
+ _node_info_to_bq_node_metadata_key_map.items()
+ })
+ scenario_result['nodeMetadata'] = node_metadata
+
+
argp = argparse.ArgumentParser(description='Upload result to big query.')
argp.add_argument('--bq_result_table',
required=True,
@@ -227,6 +253,10 @@
default='metadata.json',
type=str,
help='Metadata file to upload.')
+argp.add_argument('--node_info_file_to_upload',
+ default='node_info.json',
+ type=str,
+ help='Node information file to upload.')
argp.add_argument('--file_format',
choices=['scenario_result', 'netperf_latency_csv'],
default='scenario_result',
@@ -242,6 +272,8 @@
else:
_upload_scenario_result_to_bigquery(dataset_id, table_id,
args.file_to_upload,
- args.metadata_file_to_upload)
-print('Successfully uploaded %s and %s to BigQuery.\n' %
- (args.file_to_upload, args.metadata_file_to_upload))
+ args.metadata_file_to_upload,
+ args.node_info_file_to_upload)
+print('Successfully uploaded %s, %s and %s to BigQuery.\n' %
+ (args.file_to_upload, args.metadata_file_to_upload,
+ args.node_info_file_to_upload))
diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json
index 55b9870..095f6c9 100644
--- a/tools/run_tests/performance/scenario_result_schema.json
+++ b/tools/run_tests/performance/scenario_result_schema.json
@@ -1900,5 +1900,78 @@
"mode": "NULLABLE",
"name": "testMetadata",
"type": "STRING"
+ },
+ {
+ "fields": [
+ {
+ "fields": [
+ {
+ "mode": "NULLABLE",
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "podIP",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "nodeName",
+ "type": "STRING"
+ }
+ ],
+ "mode": "NULLABLE",
+ "name": "driver",
+ "type": "RECORD"
+ },
+ {
+ "fields": [
+ {
+ "mode": "NULLABLE",
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "podIP",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "nodeName",
+ "type": "STRING"
+ }
+ ],
+ "mode": "REPEATED",
+ "name": "servers",
+ "type": "RECORD"
+ },
+ {
+ "fields": [
+ {
+ "mode": "NULLABLE",
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "podIP",
+ "type": "STRING"
+ },
+ {
+ "mode": "NULLABLE",
+ "name": "nodeName",
+ "type": "STRING"
+ }
+ ],
+ "mode": "REPEATED",
+ "name": "clients",
+ "type": "RECORD"
+ }
+ ],
+ "mode": "NULLABLE",
+ "name": "nodeMetadata",
+ "type": "RECORD"
}
]
