| #!/usr/bin/env python3 |
| # |
| # Copyright (C) 2023 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ |
| Generate the SBOM of the current target product in SPDX format. |
| Usage example: |
| generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \ |
| --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \ |
| --product_out_dir=out/target/product/vsoc_x86_64 \ |
| --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ |
| --product_mfr=Google |
| """ |
| |
| import argparse |
| import csv |
| import datetime |
| import google.protobuf.text_format as text_format |
| import hashlib |
| import json |
| import os |
| import metadata_file_pb2 |
| |
| # Common |
| SPDXID = 'SPDXID' |
| SPDX_VERSION = 'SPDXVersion' |
| DATA_LICENSE = 'DataLicense' |
| DOCUMENT_NAME = 'DocumentName' |
| DOCUMENT_NAMESPACE = 'DocumentNamespace' |
| CREATED = 'Created' |
| CREATOR = 'Creator' |
| EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef' |
| |
| # Package |
| PACKAGE_NAME = 'PackageName' |
| PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation' |
| PACKAGE_VERSION = 'PackageVersion' |
| PACKAGE_SUPPLIER = 'PackageSupplier' |
| FILES_ANALYZED = 'FilesAnalyzed' |
| PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode' |
| PACKAGE_EXTERNAL_REF = 'ExternalRef' |
| # Package license |
| PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded' |
| PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles' |
| PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared' |
| PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments' |
| |
| # File |
| FILE_NAME = 'FileName' |
| FILE_CHECKSUM = 'FileChecksum' |
| # File license |
| FILE_LICENSE_CONCLUDED = 'LicenseConcluded' |
| FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile' |
| FILE_LICENSE_COMMENTS = 'LicenseComments' |
| FILE_COPYRIGHT_TEXT = 'FileCopyrightText' |
| FILE_NOTICE = 'FileNotice' |
| FILE_ATTRIBUTION_TEXT = 'FileAttributionText' |
| |
| # Relationship |
| RELATIONSHIP = 'Relationship' |
| REL_DESCRIBES = 'DESCRIBES' |
| REL_VARIANT_OF = 'VARIANT_OF' |
| REL_GENERATED_FROM = 'GENERATED_FROM' |
| |
| # Package type |
| PKG_SOURCE = 'SOURCE' |
| PKG_UPSTREAM = 'UPSTREAM' |
| PKG_PREBUILT = 'PREBUILT' |
| |
| # Security tag |
| NVD_CPE23 = 'NVD-CPE2.3:' |
| |
| # Report |
| ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' |
| ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' |
| ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' |
| ISSUE_UNKNOWN_SECURITY_TAG_TYPE = "Unknown security tag type:" |
| INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' |
| |
| |
| def get_args(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') |
| parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') |
| parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.') |
| parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.') |
| parser.add_argument('--build_version', required=True, help='The build version.') |
| parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') |
| parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') |
| parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module') |
| |
| return parser.parse_args() |
| |
| |
| def log(*info): |
| if args.verbose: |
| for i in info: |
| print(i) |
| |
| |
| def new_doc_header(doc_id): |
| return { |
| SPDX_VERSION: 'SPDX-2.3', |
| DATA_LICENSE: 'CC0-1.0', |
| SPDXID: doc_id, |
| DOCUMENT_NAME: args.build_version, |
| DOCUMENT_NAMESPACE: 'https://www.google.com/sbom/spdx/android/' + args.build_version, |
| CREATOR: 'Organization: Google, LLC', |
| CREATED: '<timestamp>', |
| EXTERNAL_DOCUMENT_REF: [], |
| } |
| |
| |
| def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]): |
| package = { |
| PACKAGE_NAME: name, |
| SPDXID: id, |
| PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE', |
| FILES_ANALYZED: files_analyzed, |
| } |
| if version: |
| package[PACKAGE_VERSION] = version |
| if supplier: |
| package[PACKAGE_SUPPLIER] = 'Organization: ' + supplier |
| if external_refs: |
| package[PACKAGE_EXTERNAL_REF] = external_refs |
| |
| return package |
| |
| |
| def new_file_record(id, name, checksum): |
| return { |
| FILE_NAME: name, |
| SPDXID: id, |
| FILE_CHECKSUM: checksum |
| } |
| |
| |
| def encode_for_spdxid(s): |
| """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-""" |
| result = '' |
| for c in s: |
| if c.isalnum() or c in '.-': |
| result += c |
| elif c in '_@/': |
| result += '-' |
| else: |
| result += '0x' + c.encode('utf-8').hex() |
| |
| return result.lstrip('-') |
| |
| |
| def new_package_id(package_name, type): |
| return 'SPDXRef-{}-{}'.format(type, encode_for_spdxid(package_name)) |
| |
| |
| def new_external_doc_ref(package_name, sbom_url, sbom_checksum): |
| doc_ref_id = 'DocumentRef-{}-{}'.format(PKG_UPSTREAM, encode_for_spdxid(package_name)) |
| return '{}: {} {} {}'.format(EXTERNAL_DOCUMENT_REF, doc_ref_id, sbom_url, sbom_checksum), doc_ref_id |
| |
| |
| def new_file_id(file_path): |
| return 'SPDXRef-' + encode_for_spdxid(file_path) |
| |
| |
| def new_relationship_record(id1, relationship, id2): |
| return '{}: {} {} {}'.format(RELATIONSHIP, id1, relationship, id2) |
| |
| |
| def checksum(file_path): |
| file_path = args.product_out_dir + '/' + file_path |
| h = hashlib.sha1() |
| if os.path.islink(file_path): |
| h.update(os.readlink(file_path).encode('utf-8')) |
| else: |
| with open(file_path, "rb") as f: |
| h.update(f.read()) |
| return "SHA1: " + h.hexdigest() |
| |
| |
| def is_soong_prebuilt_module(file_metadata): |
| return file_metadata['soong_module_type'] and file_metadata['soong_module_type'] in [ |
| 'android_app_import', 'android_library_import', 'cc_prebuilt_binary', 'cc_prebuilt_library', |
| 'cc_prebuilt_library_headers', 'cc_prebuilt_library_shared', 'cc_prebuilt_library_static', 'cc_prebuilt_object', |
| 'dex_import', 'java_import', 'java_sdk_library_import', 'java_system_modules_import', |
| 'libclang_rt_prebuilt_library_static', 'libclang_rt_prebuilt_library_shared', 'llvm_prebuilt_library_static', |
| 'ndk_prebuilt_object', 'ndk_prebuilt_shared_stl', 'nkd_prebuilt_static_stl', 'prebuilt_apex', |
| 'prebuilt_bootclasspath_fragment', 'prebuilt_dsp', 'prebuilt_firmware', 'prebuilt_kernel_modules', |
| 'prebuilt_rfsa', 'prebuilt_root', 'rust_prebuilt_dylib', 'rust_prebuilt_library', 'rust_prebuilt_rlib', |
| 'vndk_prebuilt_shared', |
| |
| # 'android_test_import', |
| # 'cc_prebuilt_test_library_shared', |
| # 'java_import_host', |
| # 'java_test_import', |
| # 'llvm_host_prebuilt_library_shared', |
| # 'prebuilt_apis', |
| # 'prebuilt_build_tool', |
| # 'prebuilt_defaults', |
| # 'prebuilt_etc', |
| # 'prebuilt_etc_host', |
| # 'prebuilt_etc_xml', |
| # 'prebuilt_font', |
| # 'prebuilt_hidl_interfaces', |
| # 'prebuilt_platform_compat_config', |
| # 'prebuilt_stubs_sources', |
| # 'prebuilt_usr_share', |
| # 'prebuilt_usr_share_host', |
| # 'soong_config_module_type_import', |
| ] |
| |
| |
| def is_source_package(file_metadata): |
| module_path = file_metadata['module_path'] |
| return module_path.startswith('external/') and not is_prebuilt_package(file_metadata) |
| |
| |
| def is_prebuilt_package(file_metadata): |
| module_path = file_metadata['module_path'] |
| if module_path: |
| return (module_path.startswith('prebuilts/') or |
| is_soong_prebuilt_module(file_metadata) or |
| file_metadata['is_prebuilt_make_module']) |
| |
| kernel_module_copy_files = file_metadata['kernel_module_copy_files'] |
| if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'): |
| return True |
| |
| return False |
| |
| |
| def get_source_package_info(file_metadata, metadata_file_path): |
| if not metadata_file_path: |
| return file_metadata['module_path'], [] |
| |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| external_refs = [] |
| for tag in metadata_proto.third_party.security.tag: |
| if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): |
| external_refs.append("{}: SECURITY cpe23Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23))) |
| elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): |
| external_refs.append("{}: SECURITY cpe22Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23))) |
| |
| if metadata_proto.name: |
| return metadata_proto.name, external_refs |
| else: |
| return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name |
| |
| |
| def get_prebuilt_package_name(file_metadata, metadata_file_path): |
| name = None |
| if metadata_file_path: |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| if metadata_proto.name: |
| name = metadata_proto.name |
| else: |
| name = metadata_file_path |
| elif file_metadata['module_path']: |
| name = file_metadata['module_path'] |
| elif file_metadata['kernel_module_copy_files']: |
| src_path = file_metadata['kernel_module_copy_files'].split(':')[0] |
| name = os.path.dirname(src_path) |
| |
| return name.removeprefix('prebuilts/').replace('/', '-') |
| |
| |
| def get_metadata_file_path(file_metadata): |
| metadata_path = '' |
| if file_metadata['module_path']: |
| metadata_path = file_metadata['module_path'] |
| elif file_metadata['kernel_module_copy_files']: |
| metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) |
| |
| while metadata_path and not os.path.exists(metadata_path + '/METADATA'): |
| metadata_path = os.path.dirname(metadata_path) |
| |
| return metadata_path |
| |
| |
| def get_package_version(metadata_file_path): |
| if not metadata_file_path: |
| return None |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| return metadata_proto.third_party.version |
| |
| |
| def get_package_homepage(metadata_file_path): |
| if not metadata_file_path: |
| return None |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| if metadata_proto.third_party.homepage: |
| return metadata_proto.third_party.homepage |
| for url in metadata_proto.third_party.url: |
| if url.type == metadata_file_pb2.URL.Type.HOMEPAGE: |
| return url.value |
| |
| return None |
| |
| |
| def get_package_download_location(metadata_file_path): |
| if not metadata_file_path: |
| return None |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| if metadata_proto.third_party.url: |
| urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type) |
| if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE: |
| return urls[0].value |
| elif len(urls) > 1: |
| return urls[1].value |
| |
| return None |
| |
| |
| def get_sbom_fragments(installed_file_metadata, metadata_file_path): |
| external_doc_ref = None |
| packages = [] |
| relationships = [] |
| |
| # Info from METADATA file |
| homepage = get_package_homepage(metadata_file_path) |
| version = get_package_version(metadata_file_path) |
| download_location = get_package_download_location(metadata_file_path) |
| |
| if is_source_package(installed_file_metadata): |
| # Source fork packages |
| name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) |
| source_package_id = new_package_id(name, PKG_SOURCE) |
| source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr, |
| external_refs=external_refs) |
| |
| upstream_package_id = new_package_id(name, PKG_UPSTREAM) |
| upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location) |
| packages += [source_package, upstream_package] |
| relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id)) |
| elif is_prebuilt_package(installed_file_metadata): |
| # Prebuilt fork packages |
| name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) |
| prebuilt_package_id = new_package_id(name, PKG_PREBUILT) |
| prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr) |
| packages.append(prebuilt_package) |
| |
| if metadata_file_path: |
| metadata_proto = metadata_file_protos[metadata_file_path] |
| if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref': |
| sbom_url = metadata_proto.third_party.sbom_ref.url |
| sbom_checksum = metadata_proto.third_party.sbom_ref.checksum |
| upstream_element_id = metadata_proto.third_party.sbom_ref.element_id |
| if sbom_url and sbom_checksum and upstream_element_id: |
| external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum) |
| relationships.append( |
| new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id)) |
| |
| return external_doc_ref, packages, relationships |
| |
| |
| def generate_package_verification_code(files): |
| checksums = [file[FILE_CHECKSUM] for file in files] |
| checksums.sort() |
| h = hashlib.sha1() |
| h.update(''.join(checksums).encode(encoding='utf-8')) |
| return h.hexdigest() |
| |
| |
| def write_record(f, record): |
| if record.__class__.__name__ == 'dict': |
| for k, v in record.items(): |
| if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF: |
| for ref in v: |
| f.write(ref + '\n') |
| else: |
| f.write('{}: {}\n'.format(k, v)) |
| elif record.__class__.__name__ == 'str': |
| f.write(record + '\n') |
| f.write('\n') |
| |
| |
| def write_tagvalue_sbom(all_records): |
| with open(args.output_file, 'w', encoding="utf-8") as output_file: |
| for rec in all_records: |
| write_record(output_file, rec) |
| |
| |
| def write_json_sbom(all_records, product_package_id): |
| doc = {} |
| product_package = None |
| for r in all_records: |
| if r.__class__.__name__ == 'dict': |
| if DOCUMENT_NAME in r: # Doc header |
| doc['spdxVersion'] = r[SPDX_VERSION] |
| doc['dataLicense'] = r[DATA_LICENSE] |
| doc[SPDXID] = r[SPDXID] |
| doc['name'] = r[DOCUMENT_NAME] |
| doc['documentNamespace'] = r[DOCUMENT_NAMESPACE] |
| doc['creationInfo'] = { |
| 'creators': [r[CREATOR]], |
| 'created': r[CREATED], |
| } |
| doc['externalDocumentRefs'] = [] |
| for ref in r[EXTERNAL_DOCUMENT_REF]: |
| # ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx' |
| fields = ref.split(' ') |
| doc_ref = { |
| 'externalDocumentId': fields[1], |
| 'spdxDocument': fields[2], |
| 'checksum': { |
| 'algorithm': fields[3][:-1], |
| 'checksumValue': fields[4] |
| } |
| } |
| doc['externalDocumentRefs'].append(doc_ref) |
| doc['documentDescribes'] = [] |
| doc['packages'] = [] |
| doc['files'] = [] |
| doc['relationships'] = [] |
| |
| elif PACKAGE_NAME in r: # packages |
| package = { |
| 'name': r[PACKAGE_NAME], |
| SPDXID: r[SPDXID], |
| 'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION], |
| 'filesAnalyzed': r[FILES_ANALYZED] == "true" |
| } |
| if PACKAGE_VERSION in r: |
| package['versionInfo'] = r[PACKAGE_VERSION] |
| if PACKAGE_SUPPLIER in r: |
| package['supplier'] = r[PACKAGE_SUPPLIER] |
| if PACKAGE_VERIFICATION_CODE in r: |
| package['packageVerificationCode'] = { |
| 'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE] |
| } |
| if PACKAGE_EXTERNAL_REF in r: |
| package['externalRefs'] = [] |
| for ref in r[PACKAGE_EXTERNAL_REF]: |
| # ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4' |
| fields = ref.split(' ') |
| ext_ref = { |
| 'referenceCategory': fields[1], |
| 'referenceType': fields[2], |
| 'referenceLocator': fields[3], |
| } |
| package['externalRefs'].append(ext_ref) |
| |
| doc['packages'].append(package) |
| if r[SPDXID] == product_package_id: |
| product_package = package |
| product_package['hasFiles'] = [] |
| |
| elif FILE_NAME in r: # files |
| file = { |
| 'fileName': r[FILE_NAME], |
| SPDXID: r[SPDXID] |
| } |
| checksum = r[FILE_CHECKSUM].split(': ') |
| file['checksums'] = [{ |
| 'algorithm': checksum[0], |
| 'checksumValue': checksum[1], |
| }] |
| doc['files'].append(file) |
| product_package['hasFiles'].append(r[SPDXID]) |
| |
| elif r.__class__.__name__ == 'str': |
| if r.startswith(RELATIONSHIP): |
| # r is 'Relationship: <spdxid> <relationship> <spdxid>' |
| fields = r.split(' ') |
| rel = { |
| 'spdxElementId': fields[1], |
| 'relatedSpdxElement': fields[3], |
| 'relationshipType': fields[2], |
| } |
| if fields[2] == REL_DESCRIBES: |
| doc['documentDescribes'].append(fields[3]) |
| else: |
| doc['relationships'].append(rel) |
| |
| with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file: |
| output_file.write(json.dumps(doc, indent=4)) |
| |
| |
| def save_report(report): |
| prefix, _ = os.path.splitext(args.output_file) |
| with open(prefix + '-gen-report.txt', 'w', encoding="utf-8") as report_file: |
| for type, issues in report.items(): |
| report_file.write(type + '\n') |
| for issue in issues: |
| report_file.write('\t' + issue + '\n') |
| report_file.write('\n') |
| |
| |
| def sort_rels(rel): |
| # rel = 'Relationship file_id GENERATED_FROM package_id' |
| fields = rel.split(' ') |
| return fields[3] + fields[1] |
| |
| |
| # Validate the metadata generated by Make for installed files and report if there is no metadata. |
| def installed_file_has_metadata(installed_file_metadata, report): |
| installed_file = installed_file_metadata['installed_file'] |
| module_path = installed_file_metadata['module_path'] |
| product_copy_files = installed_file_metadata['product_copy_files'] |
| kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] |
| is_platform_generated = installed_file_metadata['is_platform_generated'] |
| |
| if (not module_path and |
| not product_copy_files and |
| not kernel_module_copy_files and |
| not is_platform_generated and |
| not installed_file.endswith('.fsv_meta')): |
| report[ISSUE_NO_METADATA].append(installed_file) |
| return False |
| |
| return True |
| |
| |
| def report_metadata_file(metadata_file_path, installed_file_metadata, report): |
| if metadata_file_path: |
| report[INFO_METADATA_FOUND_FOR_PACKAGE].append( |
| "installed_file: {}, module_path: {}, METADATA file: {}".format( |
| installed_file_metadata['installed_file'], |
| installed_file_metadata['module_path'], |
| metadata_file_path + '/METADATA')) |
| |
| package_metadata = metadata_file_pb2.Metadata() |
| with open(metadata_file_path + '/METADATA', "rt") as f: |
| text_format.Parse(f.read(), package_metadata) |
| |
| if not metadata_file_path in metadata_file_protos: |
| metadata_file_protos[metadata_file_path] = package_metadata |
| if not package_metadata.name: |
| report[ISSUE_METADATA_FILE_INCOMPLETE].append('{} does not has "name"'.format(metadata_file_path + '/METADATA')) |
| |
| if not package_metadata.third_party.version: |
| report[ISSUE_METADATA_FILE_INCOMPLETE].append( |
| '{} does not has "third_party.version"'.format(metadata_file_path + '/METADATA')) |
| |
| for tag in package_metadata.third_party.security.tag: |
| if not tag.startswith(NVD_CPE23): |
| report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append( |
| "Unknown security tag type: {} in {}".format(tag, metadata_file_path + '/METADATA')) |
| else: |
| report[ISSUE_NO_METADATA_FILE].append( |
| "installed_file: {}, module_path: {}".format( |
| installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) |
| |
| |
| def generate_fragment(): |
| with open(args.metadata, newline='') as sbom_metadata_file: |
| reader = csv.DictReader(sbom_metadata_file) |
| for installed_file_metadata in reader: |
| installed_file = installed_file_metadata['installed_file'] |
| if args.output_file != args.product_out_dir + installed_file + ".spdx": |
| continue |
| |
| module_path = installed_file_metadata['module_path'] |
| package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT) |
| package = new_package_record(package_id, module_path, args.build_version, args.product_mfr) |
| file_id = new_file_id(installed_file) |
| file = new_file_record(file_id, installed_file, checksum(installed_file)) |
| relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id) |
| records = [package, file, relationship] |
| write_tagvalue_sbom(records) |
| break |
| |
| |
| def main(): |
| global args |
| args = get_args() |
| log("Args:", vars(args)) |
| |
| if args.unbundled: |
| generate_fragment() |
| return |
| |
| global metadata_file_protos |
| metadata_file_protos = {} |
| |
| doc_id = 'SPDXRef-DOCUMENT' |
| doc_header = new_doc_header(doc_id) |
| |
| product_package_id = 'SPDXRef-PRODUCT' |
| product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr, |
| files_analyzed='true') |
| |
| platform_package_id = 'SPDXRef-PLATFORM' |
| platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr) |
| |
| # Report on some issues and information |
| report = { |
| ISSUE_NO_METADATA: [], |
| ISSUE_NO_METADATA_FILE: [], |
| ISSUE_METADATA_FILE_INCOMPLETE: [], |
| ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], |
| INFO_METADATA_FOUND_FOR_PACKAGE: [] |
| } |
| |
| # Scan the metadata in CSV file and create the corresponding package and file records in SPDX |
| product_files = [] |
| package_ids = [] |
| package_records = [] |
| rels_file_gen_from = [] |
| with open(args.metadata, newline='') as sbom_metadata_file: |
| reader = csv.DictReader(sbom_metadata_file) |
| for installed_file_metadata in reader: |
| installed_file = installed_file_metadata['installed_file'] |
| module_path = installed_file_metadata['module_path'] |
| product_copy_files = installed_file_metadata['product_copy_files'] |
| kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] |
| |
| if not installed_file_has_metadata(installed_file_metadata, report): |
| continue |
| |
| file_id = new_file_id(installed_file) |
| product_files.append(new_file_record(file_id, installed_file, checksum(installed_file))) |
| |
| if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): |
| metadata_file_path = get_metadata_file_path(installed_file_metadata) |
| report_metadata_file(metadata_file_path, installed_file_metadata, report) |
| |
| # File from source fork packages or prebuilt fork packages |
| external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path) |
| if len(pkgs) > 0: |
| if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]: |
| doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref) |
| for p in pkgs: |
| if not p[SPDXID] in package_ids: |
| package_ids.append(p[SPDXID]) |
| package_records.append(p) |
| for rel in rels: |
| if not rel in package_records: |
| package_records.append(rel) |
| fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package |
| rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id)) |
| elif module_path or installed_file_metadata['is_platform_generated']: |
| # File from PLATFORM package |
| rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) |
| elif product_copy_files: |
| # Format of product_copy_files: <source path>:<dest path> |
| src_path = product_copy_files.split(':')[0] |
| # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, |
| # so process them as files from PLATFORM package |
| rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) |
| elif installed_file.endswith('.fsv_meta'): |
| # See build/make/core/Makefile:2988 |
| rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) |
| elif kernel_module_copy_files.startswith('ANDROID-GEN'): |
| # For the four files generated for _dlkm, _ramdisk partitions |
| # See build/make/core/Makefile:323 |
| rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) |
| |
| product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files) |
| |
| all_records = [ |
| doc_header, |
| product_package, |
| new_relationship_record(doc_id, REL_DESCRIBES, product_package_id), |
| ] |
| all_records += product_files |
| all_records.append(platform_package) |
| all_records += package_records |
| rels_file_gen_from.sort(key=sort_rels) |
| all_records += rels_file_gen_from |
| |
| # Save SBOM records to output file |
| doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') |
| write_tagvalue_sbom(all_records) |
| if args.json: |
| write_json_sbom(all_records, product_package_id) |
| |
| save_report(report) |
| |
| |
| if __name__ == '__main__': |
| main() |
