diff --git a/data/SPDXRdfExample.rdf b/data/SPDXRdfExample.rdf index 4ada369ba..4fe4484e5 100644 --- a/data/SPDXRdfExample.rdf +++ b/data/SPDXRdfExample.rdf @@ -3,7 +3,8 @@ xmlns:j.0="http://usefulinc.com/ns/doap#" xmlns="http://spdx.org/rdf/terms#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"> - + + Sample_Document-V2.1 2010-02-03T00:00:00Z @@ -14,6 +15,18 @@ SPDX-2.1 + + + DocumentRef-spdx-tool-2.1 + + + + d6a770ba38583ed4bb4525bd96e50461655d2759 + + + + + diff --git a/data/SPDXSimpleTag.tag b/data/SPDXSimpleTag.tag index 8e7e16ff7..db1c12ca9 100644 --- a/data/SPDXSimpleTag.tag +++ b/data/SPDXSimpleTag.tag @@ -1,7 +1,11 @@ # Document info SPDXVersion: SPDX-2.1 DataLicense: CC0-1.0 +DocumentName: Sample_Document-V2.1 +SPDXID: SPDXRef-DOCUMENT +DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: Sample Comment +ExternalDocumentRef:DocumentRef-spdx-tool-2.1 https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 # Creation info Creator: Person: Bob (bob@example.com) diff --git a/data/SPDXTagExample.tag b/data/SPDXTagExample.tag index 25261122b..b471a70b2 100644 --- a/data/SPDXTagExample.tag +++ b/data/SPDXTagExample.tag @@ -1,5 +1,8 @@ SPDXVersion: SPDX-2.1 DataLicense: CC0-1.0 +DocumentName: Sample_Document-V2.1 +SPDXID: SPDXRef-DOCUMENT +DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: This is a sample spreadsheet ## Creation Information diff --git a/spdx/document.py b/spdx/document.py index 7d307fc84..d526a8496 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -19,6 +19,78 @@ from spdx import config +@total_ordering +class ExternalDocumentRef(object): + """ + External Document References entity that contains the following fields : + - external_document_id: A unique string containing letters, numbers, '.', + '-' or '+'. + - spdx_document_uri: The unique ID of the SPDX document being referenced. + - check_sum: The checksum of the referenced SPDX document. + """ + + def __init__(self, external_document_id=None, spdx_document_uri=None, + check_sum=None): + self.external_document_id = external_document_id + self.spdx_document_uri = spdx_document_uri + self.check_sum = check_sum + + def __eq__(self, other): + return ( + isinstance(other, ExternalDocumentRef) + and self.external_document_id == other.external_document_id + and self.spdx_document_uri == other.spdx_document_uri + and self.check_sum == other.check_sum + ) + + def __lt__(self, other): + return ( + (self.external_document_id, self.spdx_document_uri, + self.check_sum) < + (other.external_document_id, other.spdx_document_uri, + other.check_sum,) + ) + + def validate(self, messages=None): + """ + Validate all fields of the ExternalDocumentRef class and update the + messages list with user friendly error messages for display. + """ + messages = messages if messages is not None else [] + + return (self.validate_ext_doc_id(messages) and + self.validate_spdx_doc_uri(messages) and + self.validate_chksum(messages) + ) + + def validate_ext_doc_id(self, messages=None): + messages = messages if messages is not None else [] + + if self.external_document_id: + return True + else: + messages.append('ExternalDocumentRef has no External Document ID.') + return False + + def validate_spdx_doc_uri(self, messages=None): + messages = messages if messages is not None else [] + + if self.spdx_document_uri: + return True + else: + messages.append('ExternalDocumentRef has no SPDX Document URI.') + return False + + def validate_chksum(self, messages=None): + messages = messages if messages is not None else [] + + if self.check_sum: + return True + else: + messages.append('ExternalDocumentRef has no Checksum.') + return False + + def _add_parens(required, text): """ Add parens around a license expression if `required` is True, otherwise @@ -189,7 +261,13 @@ class Document(object): Represent an SPDX document with these fields: - version: Spec version. Mandatory, one - Type: Version. - data_license: SPDX-Metadata license. Mandatory, one. Type: License. + - name: Name of the document. Mandatory, one. Type: str. + - spdx_id: SPDX Identifier for the document to refer to itself in + relationship to other elements. Mandatory, one. Type: str. + - ext_document_references: External SPDX documents referenced within the + given SPDX document. Optional, one or many. Type: ExternalDocumentRef - comment: Comments on the SPDX file, optional one. Type: str + - namespace: SPDX document specific namespace. Mandatory, one. Type: str - creation_info: SPDX file creation info. Mandatory, one. Type: CreationInfo - package: Package described by this document. Mandatory, one. Type: Package - extracted_licenses: List of licenses extracted that are not part of the @@ -198,12 +276,17 @@ class Document(object): Type: Review. """ - def __init__(self, version=None, data_license=None, comment=None, package=None): + def __init__(self, version=None, data_license=None, name=None, spdx_id=None, + namespace=None, comment=None, package=None): # avoid recursive impor from spdx.creationinfo import CreationInfo self.version = version self.data_license = data_license + self.name = name + self.spdx_id = spdx_id + self.ext_document_references = [] self.comment = comment + self.namespace = namespace self.creation_info = CreationInfo() self.package = package self.extracted_licenses = [] @@ -215,6 +298,9 @@ def add_review(self, review): def add_extr_lic(self, lic): self.extracted_licenses.append(lic) + def add_ext_document_reference(self, ext_doc_ref): + self.ext_document_references.append(ext_doc_ref) + @property def files(self): return self.package.files @@ -237,11 +323,14 @@ def validate(self, messages=None): return (self.validate_version(messages) and self.validate_data_lics(messages) + and self.validate_name(messages) + and self.validate_spdx_id(messages) + and self.validate_namespace(messages) + and self.validate_ext_document_references(messages) and self.validate_creation_info(messages) and self.validate_package(messages) and self.validate_extracted_licenses(messages) - and self.validate_reviews(messages) - ) + and self.validate_reviews(messages)) def validate_version(self, messages=None): # FIXME: messages should be returned @@ -268,6 +357,55 @@ def validate_data_lics(self, messages=None): messages.append('Document data license must be CC0-1.0.') return False + def validate_name(self, messages=None): + # FIXME: messages should be returned + messages = messages if messages is not None else [] + + if self.name is None: + messages.append('Document has no name.') + return False + else: + return True + + def validate_namespace(self, messages=None): + # FIXME: messages should be returned + messages = messages if messages is not None else [] + + if self.namespace is None: + messages.append('Document has no namespace.') + return False + else: + return True + + def validate_spdx_id(self, messages=None): + # FIXME: messages should be returned + messages = messages if messages is not None else [] + + if self.spdx_id is None: + messages.append('Document has no SPDX Identifier.') + return False + + if self.spdx_id.endswith('SPDXRef-DOCUMENT'): + return True + else: + messages.append('Invalid Document SPDX Identifier value.') + return False + + def validate_ext_document_references(self, messages=None): + # FIXME: messages should be returned + messages = messages if messages is not None else [] + + valid = True + for doc in self.ext_document_references: + if isinstance(doc, ExternalDocumentRef): + valid = doc.validate(messages) and valid + else: + messages.append( + 'External document references must be of the type ' + 'spdx.document.ExternalDocumentRef and not ' + type(doc)) + valid = False + return valid + def validate_reviews(self, messages=None): # FIXME: messages should be returned messages = messages if messages is not None else [] diff --git a/spdx/parsers/lexers/tagvalue.py b/spdx/parsers/lexers/tagvalue.py index 55d58510a..8e26be215 100644 --- a/spdx/parsers/lexers/tagvalue.py +++ b/spdx/parsers/lexers/tagvalue.py @@ -21,7 +21,11 @@ class Lexer(object): # Top level fields 'SPDXVersion': 'DOC_VERSION', 'DataLicense': 'DOC_LICENSE', + 'DocumentName': 'DOC_NAME', + 'SPDXID': 'DOC_SPDX_ID', 'DocumentComment': 'DOC_COMMENT', + 'DocumentNamespace': 'DOC_NAMESPACE', + 'ExternalDocumentRef': 'EXT_DOC_REF', # Creation info 'Creator': 'CREATOR', 'Created': 'CREATED', @@ -83,7 +87,8 @@ class Lexer(object): tokens = ['TEXT', 'TOOL_VALUE', 'UNKNOWN_TAG', 'ORG_VALUE', 'PERSON_VALUE', - 'DATE', 'LINE', 'CHKSUM'] + list(reserved.values()) + 'DATE', 'LINE', 'CHKSUM', 'DOC_REF_ID', + 'DOC_URI', 'EXT_DOC_REF_CHKSUM'] + list(reserved.values()) def t_text(self, t): r':\s*' @@ -112,6 +117,21 @@ def t_CHKSUM(self, t): t.value = t.value[1:].strip() return t + def t_DOC_REF_ID(self, t): + r':\s*DocumentRef-([A-Za-z0-9\+\.\-]+)' + t.value = t.value[1:].strip() + return t + + def t_DOC_URI(self, t): + r'\s*((ht|f)tps?:\/\/\S*)' + t.value = t.value.strip() + return t + + def t_EXT_DOC_REF_CHKSUM(self, t): + r'\s*SHA1:\s*[a-f0-9]{40,40}' + t.value = t.value[1:].strip() + return t + def t_TOOL_VALUE(self, t): r':\s*Tool:.+' t.value = t.value[1:].strip() diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index 7da300990..dac52d152 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -33,9 +33,14 @@ ERROR_MESSAGES = { 'DOC_VERS_VALUE': 'Invalid specVersion \'{0}\' must be SPDX-M.N where M and N are numbers.', 'DOC_D_LICS': 'Invalid dataLicense \'{0}\' must be http://spdx.org/licenses/CC0-1.0.', + 'DOC_SPDX_ID_VALUE': 'Invalid SPDXID value, SPDXID must be the document namespace appended ' + 'by "#SPDXRef-DOCUMENT", line: {0}', + 'DOC_NAMESPACE_VALUE': 'Invalid DocumentNamespace value {0}, must contain a scheme (e.g. "https:") ' + 'and should not contain the "#" delimiter.', 'LL_VALUE': 'Invalid licenseListVersion \'{0}\' must be of the format N.N where N is a number', 'CREATED_VALUE': 'Invalid created value \'{0}\' must be date in ISO 8601 format.', 'CREATOR_VALUE': 'Invalid creator value \'{0}\' must be Organization, Tool or Person.', + 'EXT_DOC_REF_VALUE': 'Failed to extract {0} from ExternalDocumentRef.', 'PKG_SUPPL_VALUE': 'Invalid package supplier value \'{0}\' must be Organization, Person or NOASSERTION.', 'PKG_ORIGINATOR_VALUE': 'Invalid package supplier value \'{0}\' must be Organization, Person or NOASSERTION.', 'PKG_DOWN_LOC': 'Invalid package download location value \'{0}\' must be a url or NONE or NOASSERTION', @@ -742,6 +747,9 @@ def parse(self, fil): for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['SpdxDocument'])): self.parse_doc_fields(s) + for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['ExternalDocumentRef'])): + self.parse_ext_doc_ref(s) + for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['CreationInfo'])): self.parse_creation_info(s) @@ -799,7 +807,20 @@ def parse_creation_info(self, ci_term): self.value_error('LL_VALUE', o) def parse_doc_fields(self, doc_term): - """Parses the version, data license and comment.""" + """Parses the version, data license, name, SPDX Identifier, namespace, + and comment.""" + try: + self.builder.set_doc_spdx_id(self.doc, doc_term) + except SPDXValueError: + self.value_error('DOC_SPDX_ID_VALUE', doc_term) + try: + if doc_term.count('#', 0, len(doc_term)) <= 1: + doc_namespace = doc_term.split('#')[0] + self.builder.set_doc_namespace(self.doc, doc_namespace) + else: + self.value_error('DOC_NAMESPACE_VALUE', doc_term) + except SPDXValueError: + self.value_error('DOC_NAMESPACE_VALUE', doc_term) for _s, _p, o in self.graph.triples((doc_term, self.spdx_namespace['specVersion'], None)): try: self.builder.set_doc_version(self.doc, six.text_type(o)) @@ -816,9 +837,50 @@ def parse_doc_fields(self, doc_term): except CardinalityError: self.more_than_one_error('dataLicense') break + for _s, _p, o in self.graph.triples( + (doc_term, self.spdx_namespace['name'], None)): + try: + self.builder.set_doc_name(self.doc, six.text_type(o)) + except CardinalityError: + self.more_than_one_error('name') + break for _s, _p, o in self.graph.triples((doc_term, RDFS.comment, None)): try: self.builder.set_doc_comment(self.doc, six.text_type(o)) except CardinalityError: self.more_than_one_error('Document comment') break + + def parse_ext_doc_ref(self, ext_doc_ref_term): + """ + Parses the External Document ID, SPDX Document URI and Checksum. + """ + for _s, _p, o in self.graph.triples( + (ext_doc_ref_term, + self.spdx_namespace['externalDocumentId'], + None)): + try: + self.builder.set_ext_doc_id(self.doc, six.text_type(o)) + except SPDXValueError: + self.value_error('EXT_DOC_REF_VALUE', 'External Document ID') + break + + for _s, _p, o in self.graph.triples( + (ext_doc_ref_term, + self.spdx_namespace['spdxDocument'], + None)): + try: + self.builder.set_spdx_doc_uri(self.doc, six.text_type(o)) + except SPDXValueError: + self.value_error('EXT_DOC_REF_VALUE', 'SPDX Document URI') + break + + for _s, _p, checksum in self.graph.triples( + (ext_doc_ref_term, self.spdx_namespace['checksum'], None)): + for _, _, value in self.graph.triples( + (checksum, self.spdx_namespace['checksumValue'], None)): + try: + self.builder.set_chksum(self.doc, six.text_type(value)) + except SPDXValueError: + self.value_error('EXT_DOC_REF_VALUE', 'Checksum') + break diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index a18ef6f15..a354d6fcc 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -23,6 +23,7 @@ from spdx.parsers.builderexceptions import OrderError from spdx.parsers.builderexceptions import SPDXValueError from spdx.parsers import tagvaluebuilders +from spdx.parsers import validations class DocBuilder(object): @@ -70,6 +71,32 @@ def set_doc_data_lic(self, doc, res): else: raise CardinalityError('Document::License') + def set_doc_name(self, doc, name): + """ + Sets the document name, raises CardinalityError if already defined. + """ + if not self.doc_name_set: + doc.name = name + self.doc_name_set = True + return True + else: + raise CardinalityError('Document::Name') + + def set_doc_spdx_id(self, doc, doc_spdx_id_line): + """Sets the document SPDX Identifier. + Raises value error if malformed value, CardinalityError + if already defined. + """ + if not self.doc_spdx_id_set: + if validations.validate_doc_spdx_id(doc_spdx_id_line): + doc.spdx_id = doc_spdx_id_line + self.doc_spdx_id_set = True + return True + else: + raise SPDXValueError('Document::SPDXID') + else: + raise CardinalityError('Document::SPDXID') + def set_doc_comment(self, doc, comment): """Sets document comment, Raises CardinalityError if comment already set. @@ -80,6 +107,21 @@ def set_doc_comment(self, doc, comment): else: raise CardinalityError('Document::Comment') + def set_doc_namespace(self, doc, namespace): + """Sets the document namespace. + Raise SPDXValueError if malformed value, CardinalityError + if already defined. + """ + if not self.doc_namespace_set: + self.doc_namespace_set = True + if validations.validate_doc_namespace(namespace): + doc.namespace = namespace + return True + else: + raise SPDXValueError('Document::Namespace') + else: + raise CardinalityError('Document::Comment') + def reset_document(self): """ Reset the internal state to allow building new document @@ -87,7 +129,24 @@ def reset_document(self): # FIXME: this state does not make sense self.doc_version_set = False self.doc_comment_set = False + self.doc_namespace_set = False self.doc_data_lics_set = False + self.doc_name_set = False + self.doc_spdx_id_set = False + + +class ExternalDocumentRefBuilder(tagvaluebuilders.ExternalDocumentRefBuilder): + + def set_chksum(self, doc, chk_sum): + """ + Sets the external document reference's check sum, if not already set. + chk_sum - The checksum value in the form of a string. + """ + if chk_sum: + doc.ext_document_references[-1].check_sum = checksum.Algorithm( + 'SHA1', chk_sum) + else: + raise SPDXValueError('ExternalDocumentRef::Checksum') class EntityBuilder(tagvaluebuilders.EntityBuilder): @@ -325,7 +384,8 @@ def add_review_comment(self, doc, comment): raise OrderError('ReviewComment') -class Builder(DocBuilder, EntityBuilder, CreationInfoBuilder, PackageBuilder, FileBuilder, ReviewBuilder): +class Builder(DocBuilder, EntityBuilder, CreationInfoBuilder, PackageBuilder, + FileBuilder, ReviewBuilder, ExternalDocumentRefBuilder): def __init__(self): super(Builder, self).__init__() diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index 67693c9e5..1961a3f72 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -39,7 +39,15 @@ 'DOC_LICENSE_VALUE_TYPE': 'DataLicense must be CC0-1.0, line: {0}', 'DOC_VERSION_VALUE': 'Invalid SPDXVersion \'{0}\' must be SPDX-M.N where M and N are numbers. Line: {1}', 'DOC_VERSION_VALUE_TYPE': 'Invalid SPDXVersion value, must be SPDX-M.N where M and N are numbers. Line: {0}', + 'DOC_NAME_VALUE': 'DocumentName must be single line of text, line: {0}', + 'DOC_SPDX_ID_VALUE': 'Invalid SPDXID value, SPDXID must be SPDXRef-DOCUMENT, line: {0}', + 'EXT_DOC_REF_VALUE': 'ExternalDocumentRef must contain External Document ID, SPDX Document URI and Checksum' + 'in the standard format, line:{0}.', 'DOC_COMMENT_VALUE_TYPE': 'DocumentComment value must be free form text between tags, line:{0}', + 'DOC_NAMESPACE_VALUE': 'Invalid DocumentNamespace value {0}, must contain a scheme (e.g. "https:") ' + 'and should not contain the "#" delimiter, line:{1}', + 'DOC_NAMESPACE_VALUE_TYPE': 'Invalid DocumentNamespace value, must contain a scheme (e.g. "https:") ' + 'and should not contain the "#" delimiter, line: {0}', 'REVIEWER_VALUE_TYPE': 'Invalid Reviewer value must be a Person, Organization or Tool. Line: {0}', 'CREATOR_VALUE_TYPE': 'Invalid Reviewer value must be a Person, Organization or Tool. Line: {0}', 'REVIEW_DATE_VALUE_TYPE': 'ReviewDate value must be date in ISO 8601 format, line: {0}', @@ -106,7 +114,11 @@ def p_start_2(self, p): def p_attrib(self, p): """attrib : spdx_version | data_lics + | doc_name + | doc_spdx_id + | ext_doc_ref | doc_comment + | doc_namespace | creator | created | creator_comment @@ -1077,6 +1089,27 @@ def p_doc_comment_2(self, p): msg = ERROR_MESSAGES['DOC_COMMENT_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) + def p_doc_namespace_1(self, p): + """doc_namespace : DOC_NAMESPACE LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_doc_namespace(self.document, value) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['DOC_NAMESPACE_VALUE'].format(p[2], p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('DocumentNamespace', p.lineno(1)) + + def p_doc_namespace_2(self, p): + """doc_namespace : DOC_NAMESPACE error""" + self.error = True + msg = ERROR_MESSAGES['DOC_NAMESPACE_VALUE_TYPE'].format(p.lineno(1)) + self.logger.log(msg) + def p_data_license_1(self, p): """data_lics : DOC_LICENSE LINE""" try: @@ -1098,6 +1131,69 @@ def p_data_license_2(self, p): msg = ERROR_MESSAGES['DOC_LICENSE_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) + def p_doc_name_1(self, p): + """doc_name : DOC_NAME LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_doc_name(self.document, value) + except CardinalityError: + self.more_than_one_error('DocumentName', p.lineno(1)) + + def p_doc_name_2(self, p): + """doc_name : DOC_NAME error""" + self.error = True + msg = ERROR_MESSAGES['DOC_NAME_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_doc_spdx_id_1(self, p): + """doc_spdx_id : DOC_SPDX_ID LINE""" + try: + if six.PY2: + value = p[2].decode(encoding='utf-8') + else: + value = p[2] + self.builder.set_doc_spdx_id(self.document, value) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['DOC_SPDX_ID_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + except CardinalityError: + self.more_than_one_error('SPDXID', p.lineno(1)) + + def p_doc_spdx_id_2(self, p): + """doc_spdx_id : DOC_SPDX_ID error""" + self.error = True + msg = ERROR_MESSAGES['DOC_SPDX_ID_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + + def p_ext_doc_refs_1(self, p): + """ext_doc_ref : EXT_DOC_REF DOC_REF_ID DOC_URI EXT_DOC_REF_CHKSUM""" + try: + if six.PY2: + doc_ref_id = p[2].decode(encoding='utf-8') + doc_uri = p[3].decode(encoding='utf-8') + ext_doc_chksum = p[4].decode(encoding='utf-8') + else: + doc_ref_id = p[2] + doc_uri = p[3] + ext_doc_chksum = p[4] + + self.builder.add_ext_doc_refs(self.document, doc_ref_id, doc_uri, + ext_doc_chksum) + except SPDXValueError: + self.error = True + msg = ERROR_MESSAGES['EXT_DOC_REF_VALUE'].format(p.lineno(2)) + self.logger.log(msg) + + def p_ext_doc_refs_2(self, p): + """ext_doc_ref : EXT_DOC_REF error""" + self.error = True + msg = ERROR_MESSAGES['EXT_DOC_REF_VALUE'].format(p.lineno(1)) + self.logger.log(msg) + def p_spdx_version_1(self, p): """spdx_version : DOC_VERSION LINE""" try: diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index 385c51c4e..e5b4da932 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -27,6 +27,7 @@ from spdx import utils from spdx import version +from spdx.document import ExternalDocumentRef from spdx.parsers.builderexceptions import CardinalityError from spdx.parsers.builderexceptions import OrderError from spdx.parsers.builderexceptions import SPDXValueError @@ -102,6 +103,32 @@ def set_doc_data_lics(self, doc, lics): else: raise CardinalityError('Document::DataLicense') + def set_doc_name(self, doc, name): + """Sets the document name. + Raises CardinalityError if already defined. + """ + if not self.doc_name_set: + doc.name = name + self.doc_name_set = True + return True + else: + raise CardinalityError('Document::Name') + + def set_doc_spdx_id(self, doc, doc_spdx_id_line): + """Sets the document SPDX Identifier. + Raises value error if malformed value, CardinalityError + if already defined. + """ + if not self.doc_spdx_id_set: + if doc_spdx_id_line == 'SPDXRef-DOCUMENT': + doc.spdx_id = doc_spdx_id_line + self.doc_spdx_id_set = True + return True + else: + raise SPDXValueError('Document::SPDXID') + else: + raise CardinalityError('Document::SPDXID') + def set_doc_comment(self, doc, comment): """Sets document comment, Raises CardinalityError if comment already set. @@ -117,12 +144,65 @@ def set_doc_comment(self, doc, comment): else: raise CardinalityError('Document::Comment') + def set_doc_namespace(self, doc, namespace): + """Sets the document namespace. + Raise SPDXValueError if malformed value, CardinalityError + if already defined. + """ + if not self.doc_namespace_set: + self.doc_namespace_set = True + if validations.validate_doc_namespace(namespace): + doc.namespace = namespace + return True + else: + raise SPDXValueError('Document::Namespace') + else: + raise CardinalityError('Document::Comment') + def reset_document(self): """Resets the state to allow building new documents""" # FIXME: this state does not make sense self.doc_version_set = False self.doc_comment_set = False + self.doc_namespace_set = False self.doc_data_lics_set = False + self.doc_name_set = False + self.doc_spdx_id_set = False + + +class ExternalDocumentRefBuilder(object): + + def set_ext_doc_id(self, doc, ext_doc_id): + """ + Sets the `external_document_id` attribute of the `ExternalDocumentRef` + object. + """ + doc.add_ext_document_reference( + ExternalDocumentRef( + external_document_id=ext_doc_id)) + + def set_spdx_doc_uri(self, doc, spdx_doc_uri): + """ + Sets the `spdx_document_uri` attribute of the `ExternalDocumentRef` + object. + """ + if validations.validate_doc_namespace(spdx_doc_uri): + doc.ext_document_references[-1].spdx_document_uri = spdx_doc_uri + else: + raise SPDXValueError('Document::ExternalDocumentRef') + + def set_chksum(self, doc, chksum): + """ + Sets the `check_sum` attribute of the `ExternalDocumentRef` + object. + """ + doc.ext_document_references[-1].check_sum = checksum_from_sha1( + chksum) + + def add_ext_doc_refs(self, doc, ext_doc_id, spdx_doc_uri, chksum): + self.set_ext_doc_id(doc, ext_doc_id) + self.set_spdx_doc_uri(doc, spdx_doc_uri) + self.set_chksum(doc, chksum) class EntityBuilder(object): @@ -942,7 +1022,8 @@ def reset_extr_lics(self): class Builder(DocBuilder, CreationInfoBuilder, EntityBuilder, ReviewBuilder, - PackageBuilder, FileBuilder, LicenseBuilder): + PackageBuilder, FileBuilder, LicenseBuilder, + ExternalDocumentRefBuilder): """SPDX document builder.""" diff --git a/spdx/parsers/validations.py b/spdx/parsers/validations.py index 11ac460ea..deaceffa8 100644 --- a/spdx/parsers/validations.py +++ b/spdx/parsers/validations.py @@ -55,6 +55,10 @@ def validate_data_lics(value): return value == 'CC0-1.0' +def validate_doc_name(value, optional=False): + return validate_tool_name(value, optional) + + def validate_pkg_supplier(value, optional=False): if optional and value is None: return True @@ -100,6 +104,26 @@ def validate_doc_comment(value, optional=False): return validate_is_free_form_text(value, optional) +def validate_doc_spdx_id(value, optional=False): + if value is None: + return optional + elif value.endswith('#SPDXRef-DOCUMENT'): + return True + else: + return False + + +def validate_doc_namespace(value, optional=False): + if value is None: + return optional + elif ((value.startswith('http://') or value.startswith( + 'https://') or + value.startswith('ftp://')) and ('#' not in value)): + return True + else: + return False + + def validate_creator(value, optional=False): if value is None: return optional diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index f6f3199bd..2047cace8 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -362,6 +362,47 @@ def create_creation_info(self): return ci_node +class ExternalDocumentRefWriter(BaseWriter): + """ + Write class spdx.external_document_ref.ExternalDocumentRef + """ + + def __init__(self, document, out): + super(ExternalDocumentRefWriter, self).__init__(document, out) + + def create_external_document_ref_node(self, ext_document_references): + """ + Add and return a creation info node to graph + """ + ext_doc_ref_node = BNode() + type_triple = (ext_doc_ref_node, RDF.type, self.spdx_namespace.ExternalDocumentRef) + self.graph.add(type_triple) + + ext_doc_id = Literal( + ext_document_references.external_document_id) + ext_doc_id_triple = ( + ext_doc_ref_node, self.spdx_namespace.externalDocumentId, ext_doc_id) + self.graph.add(ext_doc_id_triple) + + doc_uri = Literal( + ext_document_references.spdx_document_uri) + doc_uri_triple = ( + ext_doc_ref_node, self.spdx_namespace.spdxDocument, doc_uri) + self.graph.add(doc_uri_triple) + + checksum_node = self.create_checksum_node( + ext_document_references.check_sum) + self.graph.add( + (ext_doc_ref_node, self.spdx_namespace.checksum, checksum_node)) + + return ext_doc_ref_node + + def ext_doc_refs(self): + "Returns a list of review nodes" + return map(self.create_external_document_ref_node, + self.document.ext_document_references) + + class PackageWriter(LicenseWriter): """ @@ -495,7 +536,8 @@ def handle_package_has_file(self, package, package_node): self.graph.add(triple) -class Writer(CreationInfoWriter, ReviewInfoWriter, FileWriter, PackageWriter): +class Writer(CreationInfoWriter, ReviewInfoWriter, FileWriter, PackageWriter, + ExternalDocumentRefWriter): """ Warpper for other writers to write all fields of spdx.document.Document Call `write()` to start writing. @@ -512,7 +554,7 @@ def create_doc(self): """ Add and return the root document node to graph. """ - doc_node = URIRef('http://www.spdx.org/tools#SPDXANALYSIS') + doc_node = URIRef('http://www.spdx.org/tools#SPDXRef-DOCUMENT') # Doc type self.graph.add((doc_node, RDF.type, self.spdx_namespace.SpdxDocument)) # Version @@ -521,6 +563,8 @@ def create_doc(self): # Data license data_lics = URIRef(self.document.data_license.url) self.graph.add((doc_node, self.spdx_namespace.dataLicense, data_lics)) + doc_name = URIRef(self.document.name) + self.graph.add((doc_node, self.spdx_namespace.name, doc_name)) return doc_node def write(self): @@ -533,6 +577,13 @@ def write(self): review_nodes = self.reviews() for review in review_nodes: self.graph.add((doc_node, self.spdx_namespace.reviewed, review)) + # Add external document references info + ext_doc_ref_nodes = self.ext_doc_refs() + for ext_doc_ref in ext_doc_ref_nodes: + ext_doc_ref_triple = (doc_node, + self.spdx_namespace.externalDocumentRef, + ext_doc_ref) + self.graph.add(ext_doc_ref_triple) # Add extracted licenses licenses = map( self.create_extracted_license, self.document.extracted_licenses) diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 9af92ef1e..e4ca0f673 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -236,8 +236,17 @@ def write_document(document, out, validate=True): out.write('# Document Information\n\n') write_value('SPDXVersion', str(document.version), out) write_value('DataLicense', document.data_license.identifier, out) + write_value('DocumentName', document.name, out) + write_value('SPDXID', 'SPDXRef-DOCUMENT', out) + write_value('DocumentNamespace', document.namespace, out) if document.has_comment: write_text_value('DocumentComment', document.comment, out) + for doc_ref in document.ext_document_references: + doc_ref_str = ' '.join([doc_ref.external_document_id, + doc_ref.spdx_document_uri, + doc_ref.check_sum.identifier + ':' + + doc_ref.check_sum.value]) + write_value('ExternalDocumentRef', doc_ref_str, out) write_separators(out) # Write out creation info write_creation_info(document.creation_info, out) diff --git a/tests/data/doc_write/rdf-simple-plus.json b/tests/data/doc_write/rdf-simple-plus.json index 1b792534a..914f3249a 100644 --- a/tests/data/doc_write/rdf-simple-plus.json +++ b/tests/data/doc_write/rdf-simple-plus.json @@ -25,7 +25,10 @@ "ns1:specVersion": "SPDX-2.1", "ns1:dataLicense": { "@rdf:resource": "http://spdx.org/licenses/CC0-1.0" - }, + }, + "ns1:name": { + "@rdf:resource": "Sample_Document-V2.1" + }, "ns1:referencesFile": { "ns1:File": { "ns1:fileName": "./some/path/tofile", @@ -46,7 +49,7 @@ } } }, - "@rdf:about": "http://www.spdx.org/tools#SPDXANALYSIS" + "@rdf:about": "http://www.spdx.org/tools#SPDXRef-DOCUMENT" } } } \ No newline at end of file diff --git a/tests/data/doc_write/rdf-simple.json b/tests/data/doc_write/rdf-simple.json index ff436589c..a64a290b3 100644 --- a/tests/data/doc_write/rdf-simple.json +++ b/tests/data/doc_write/rdf-simple.json @@ -25,7 +25,10 @@ "ns1:specVersion": "SPDX-2.1", "ns1:dataLicense": { "@rdf:resource": "http://spdx.org/licenses/CC0-1.0" - }, + }, + "ns1:name": { + "@rdf:resource": "Sample_Document-V2.1" + }, "ns1:referencesFile": { "ns1:File": { "ns1:licenseInfoInFile": { @@ -46,7 +49,7 @@ "ns1:fileName": "./some/path/tofile" } }, - "@rdf:about": "http://www.spdx.org/tools#SPDXANALYSIS" + "@rdf:about": "http://www.spdx.org/tools#SPDXRef-DOCUMENT" } } } \ No newline at end of file diff --git a/tests/data/doc_write/tv-simple-plus.tv b/tests/data/doc_write/tv-simple-plus.tv index 77baa7d89..8e5154852 100644 --- a/tests/data/doc_write/tv-simple-plus.tv +++ b/tests/data/doc_write/tv-simple-plus.tv @@ -1,6 +1,9 @@ # Document Information SPDXVersion: SPDX-2.1 DataLicense: CC0-1.0 +DocumentName: Sample_Document-V2.1 +SPDXID: SPDXRef-DOCUMENT +DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 # Creation Info # Package PackageName: some/path diff --git a/tests/data/doc_write/tv-simple.tv b/tests/data/doc_write/tv-simple.tv index a41928039..7698203f0 100644 --- a/tests/data/doc_write/tv-simple.tv +++ b/tests/data/doc_write/tv-simple.tv @@ -1,6 +1,9 @@ # Document Information SPDXVersion: SPDX-2.1 DataLicense: CC0-1.0 +DocumentName: Sample_Document-V2.1 +SPDXID: SPDXRef-DOCUMENT +DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 # Creation Info # Package PackageName: some/path diff --git a/tests/test_builder.py b/tests/test_builder.py index 93a32dd54..003eb6407 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -63,6 +63,33 @@ def test_data_lics_cardinality(self): self.builder.set_doc_data_lics(self.document, lics_str) self.builder.set_doc_data_lics(self.document, lics_str) + def test_correct_name(self): + name_str = 'Sample_Document-V2.1' + self.builder.set_doc_name(self.document, name_str) + assert self.document.name == name_str + + @testing_utils.raises(builders.CardinalityError) + def test_name_cardinality(self): + name_str = 'Sample_Document-V2.1' + self.builder.set_doc_name(self.document, name_str) + self.builder.set_doc_name(self.document, name_str) + + def test_correct_doc_namespace(self): + doc_namespace_str = 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301' + self.builder.set_doc_namespace(self.document, doc_namespace_str) + assert self.document.namespace == doc_namespace_str + + @testing_utils.raises(builders.SPDXValueError) + def test_doc_namespace_value(self): + doc_namespace_str = 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-DOCUMENT' + self.builder.set_doc_data_lics(self.document, doc_namespace_str) + + @testing_utils.raises(builders.CardinalityError) + def test_doc_namespace_cardinality(self): + doc_namespace_str = 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301' + self.builder.set_doc_namespace(self.document, doc_namespace_str) + self.builder.set_doc_namespace(self.document, doc_namespace_str) + def test_correct_data_comment(self): comment_str = 'This is a comment.' comment_text = '' + comment_str + '' @@ -82,6 +109,43 @@ def test_comment_value(self): self.builder.set_doc_comment(self.document, comment) +class TestExternalDocumentRefBuilder(TestCase): + + def setUp(self): + self.document = Document() + self.builder = builders.ExternalDocumentRefBuilder() + + def test_external_doc_id(self): + ext_doc_id = 'DocumentRef-spdx-tool-2.1' + self.builder.set_ext_doc_id(self.document, ext_doc_id) + assert self.document.ext_document_references[-1].external_document_id == ext_doc_id + + def test_spdx_doc_uri(self): + spdx_doc_uri = 'https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301' + self.builder.set_ext_doc_id(self.document, 'DocumentRef-spdx-tool-2.1') + self.builder.set_spdx_doc_uri(self.document, spdx_doc_uri) + assert self.document.ext_document_references[-1].spdx_document_uri == spdx_doc_uri + + def test_checksum(self): + chksum = 'SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759' + chksum_val = 'd6a770ba38583ed4bb4525bd96e50461655d2759' + self.builder.set_ext_doc_id(self.document, 'DocumentRef-spdx-tool-2.1') + self.builder.set_chksum(self.document, chksum) + assert self.document.ext_document_references[-1].check_sum.value == chksum_val + + def test_add_ext_doc_refs(self): + ext_doc_id_val = 'DocumentRef-spdx-tool-2.1' + spdx_doc_uri = 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301' + chksum = 'SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759' + chksum_val = 'd6a770ba38583ed4bb4525bd96e50461655d2759' + + self.builder.add_ext_doc_refs(self.document, ext_doc_id_val, + spdx_doc_uri, chksum) + assert self.document.ext_document_references[-1].external_document_id == ext_doc_id_val + assert self.document.ext_document_references[-1].spdx_document_uri == spdx_doc_uri + assert self.document.ext_document_references[-1].check_sum.value == chksum_val + + class TestEntityBuilder(TestCase): def setUp(self): diff --git a/tests/test_document.py b/tests/test_document.py index 69c7dd4b3..4b860990e 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -22,7 +22,7 @@ from spdx.checksum import Algorithm from spdx.config import LICENSE_MAP from spdx.creationinfo import Tool -from spdx.document import Document +from spdx.document import Document, ExternalDocumentRef from spdx.document import License from spdx.file import File from spdx.package import Package @@ -61,12 +61,23 @@ def test_creation(self): data_license=License(full_name='Academic Free License v1.1', identifier='AFL-1.1') ) + document.add_ext_document_reference( + ExternalDocumentRef('DocumentRef-spdx-tool-2.1', + 'https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301', + Algorithm('SHA1', 'SOME-SHA1')) + ) assert document.comment is None assert document.version == Version(2, 1) assert document.data_license.identifier == 'AFL-1.1' + assert document.ext_document_references[-1].external_document_id == 'DocumentRef-spdx-tool-2.1' + assert document.ext_document_references[-1].spdx_document_uri == 'https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301' + assert document.ext_document_references[-1].check_sum.identifier == 'SHA1' + assert document.ext_document_references[-1].check_sum.value == 'SOME-SHA1' def test_document_validate_failures_returns_informative_messages(self): - doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) + doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'), + 'Sample_Document-V2.1', spdx_id='SPDXRef-DOCUMENT', + namespace='https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301') pack = doc.package = Package('some/path', NoAssert()) file1 = File('./some/path/tofile') file1.name = './some/path/tofile' @@ -83,7 +94,9 @@ def test_document_validate_failures_returns_informative_messages(self): assert expected == messages def test_document_is_valid_when_using_or_later_licenses(self): - doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) + doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'), + 'Sample_Document-V2.1', spdx_id='SPDXRef-DOCUMENT', + namespace='https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301') doc.creation_info.add_creator(Tool('ScanCode')) doc.creation_info.set_created_now() @@ -113,7 +126,9 @@ def test_document_is_valid_when_using_or_later_licenses(self): class TestWriters(TestCase): def _get_lgpl_doc(self, or_later=False): - doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) + doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'), + 'Sample_Document-V2.1', spdx_id='SPDXRef-DOCUMENT', + namespace='https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301') doc.creation_info.add_creator(Tool('ScanCode')) doc.creation_info.set_created_now() diff --git a/tests/test_tag_value_parser.py b/tests/test_tag_value_parser.py index 2c619af72..f62329476 100644 --- a/tests/test_tag_value_parser.py +++ b/tests/test_tag_value_parser.py @@ -35,17 +35,46 @@ def test_document(self): data = ''' SPDXVersion: SPDX-2.1 # Comment. - DocumentComment: This is a sample spreadsheet DataLicense: CC0-1.0 + DocumentName: Sample_Document-V2.1 + SPDXID: SPDXRef-DOCUMENT + DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 + DocumentComment: This is a sample spreadsheet ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2) - self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 4) - self.token_assert_helper(self.l.token(), 'TEXT', 'This is a sample spreadsheet', 4) - self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', + self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) + self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) + self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5) + self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1', 5) - self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 5) + self.token_assert_helper(self.l.token(), 'DOC_SPDX_ID', 'SPDXID', 6) + self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6) + self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE', + 'DocumentNamespace', 7) + self.token_assert_helper(self.l.token(), 'LINE', + 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301', + 7) + self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8) + self.token_assert_helper(self.l.token(), 'TEXT', 'This is a sample spreadsheet', 8) + + def test_external_document_references(self): + data = ''' + ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 + ''' + self.l.input(data) + self.token_assert_helper(self.l.token(), 'EXT_DOC_REF', + 'ExternalDocumentRef', 2) + self.token_assert_helper(self.l.token(), 'DOC_REF_ID', + 'DocumentRef-spdx-tool-2.1', 2) + self.token_assert_helper(self.l.token(), 'DOC_URI', + 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25' + '04E0-4F89-41D3-9A0C-0305E82C3301', 2) + self.token_assert_helper(self.l.token(), 'EXT_DOC_REF_CHKSUM', + 'SHA1: ' + 'd6a770ba38583ed4bb4525bd96e50461655d2759', 2) + def test_creation_info(self): data = ''' @@ -115,7 +144,10 @@ class TestParser(TestCase): document_str = '\n'.join([ 'SPDXVersion: SPDX-2.1', 'DataLicense: CC0-1.0', - 'DocumentComment: Sample Comment' + 'DocumentName: Sample_Document-V2.1', + 'SPDXID: SPDXRef-DOCUMENT', + 'DocumentComment: Sample Comment', + 'DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301' ]) creation_str = '\n'.join([ @@ -181,7 +213,10 @@ def test_doc(self): assert not error assert document.version == Version(major=2, minor=1) assert document.data_license.identifier == 'CC0-1.0' + assert document.name == 'Sample_Document-V2.1' + assert document.spdx_id == 'SPDXRef-DOCUMENT' assert document.comment == 'Sample Comment' + assert document.namespace == 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301' def test_creation_info(self): document, error = self.p.parse(self.complete_str) @@ -235,4 +270,4 @@ def test_unknown_tag(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main()