| #!/usr/bin/python -i |
| |
| import sys |
| try: |
| import urllib.request as urllib2 |
| except ImportError: |
| import urllib2 |
| from bs4 import BeautifulSoup |
| import json |
| import vuid_mapping |
| import re |
| |
| ############################# |
| # spec.py script |
| # |
| # Overview - this script is intended to generate validation error codes and message strings from the json spec file |
| # that contains all of the valid usage statements. In addition to generating the header file, it provides a number of |
| # corrollary services to aid in generating/updating the header. |
| # |
| # Ideal flow - Pull the valid usage text and IDs from the spec json, pull the IDs from the validation error database, |
| # then update the database with any new IDs from the json file and generate new database and header file. |
| # |
| # TODO: |
| # 1. When VUs go away (in error DB, but not in json) need to report them and remove from DB as deleted |
| # |
| ############################# |
| |
| |
| out_filename = "../layers/vk_validation_error_messages.h" # can override w/ '-out <filename>' option |
| db_filename = "../layers/vk_validation_error_database.txt" # can override w/ '-gendb <filename>' option |
| json_filename = "../scripts/validusage.json" # can override w/ '-json-file <filename> option |
| gen_db = False # set to True when '-gendb <filename>' option provided |
| json_compare = False # compare existing DB to json file input |
| # This is the root spec link that is used in error messages to point users to spec sections |
| #old_spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0/xhtml/vkspec.html" |
| spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html" |
| core_url = "https://www.khronos.org/registry/vulkan/specs/1.0/html/vkspec.html" |
| ext_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html" |
| # After the custom validation error message, this is the prefix for the standard message that includes the |
| # spec valid usage language as well as the link to nearest section of spec to that language |
| error_msg_prefix = "The spec valid usage text states " |
| validation_error_enum_name = "VALIDATION_ERROR_" |
| |
| def printHelp(): |
| print ("Usage: python spec.py [-out <headerfile.h>] [-gendb <databasefile.txt>] [-update] [-json-file <json_file>] [-help]") |
| print ("\n Default script behavior is to parse the specfile and generate a header of unique error enums and corresponding error messages based on the specfile.\n") |
| print (" Default specfile is from online at %s" % (spec_url)) |
| print (" Default headerfile is %s" % (out_filename)) |
| print (" Default databasefile is %s" % (db_filename)) |
| print ("\nIf '-gendb' option is specified then a database file is generated to default file or <databasefile.txt> if supplied. The database file stores") |
| print (" the list of enums and their error messages.") |
| print ("\nIf '-update' option is specified this triggers the master flow to automate updating header and database files using default db file as baseline") |
| print (" and online spec file as the latest. The default header and database files will be updated in-place for review and commit to the git repo.") |
| print ("\nIf '-json-file' option is specified, it will override the default json file location") |
| |
| def get8digithex(dec_num): |
| """Convert a decimal # into an 8-digit hex""" |
| if dec_num > 4294967295: |
| print ("ERROR: Decimal # %d can't be represented in 8 hex digits" % (dec_num)) |
| sys.exit() |
| hex_num = hex(dec_num) |
| return hex_num[2:].zfill(8) |
| |
| class Specification: |
| def __init__(self): |
| self.tree = None |
| self.error_db_dict = {} # dict of previous error values read in from database file |
| self.delimiter = '~^~' # delimiter for db file |
| # Global dicts used for tracking spec updates from old to new VUs |
| self.orig_no_link_msg_dict = {} # Pair of API,Original msg w/o spec link to ID list mapping |
| self.orig_core_msg_dict = {} # Pair of API,Original core msg (no link or section) to ID list mapping |
| self.last_mapped_id = -10 # start as negative so we don't hit an accidental sequence |
| self.orig_test_imp_enums = set() # Track old enums w/ tests and/or implementation to flag any that aren't carried fwd |
| # Dict of data from json DB |
| # Key is API,<short_msg> which leads to dict w/ following values |
| # 'ext' -> <core|<ext_name>> |
| # 'string_vuid' -> <string_vuid> |
| # 'number_vuid' -> <numerical_vuid> |
| self.json_db = {} |
| self.json_missing = 0 |
| self.struct_to_func_map = {} # Map structs to the API func that they fall under in the spec |
| self.duplicate_json_key_count = 0 |
| self.copyright = """/* THIS FILE IS GENERATED. DO NOT EDIT. */ |
| |
| /* |
| * Vulkan |
| * |
| * Copyright (c) 2016 Google Inc. |
| * Copyright (c) 2016 LunarG, Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| * Author: Tobin Ehlis <tobine@google.com> |
| */""" |
| |
| def readJSON(self): |
| """Read in JSON file""" |
| if json_filename is not None: |
| with open(json_filename) as jsf: |
| self.json_data = json.load(jsf, encoding='utf-8') |
| else: |
| response = urllib2.urlopen(json_url).read().decode('utf-8') |
| self.json_data = json.loads(response) |
| |
| def parseJSON(self): |
| """Parse JSON VUIDs into data struct""" |
| # Format of JSON file is: |
| # "API": { "core|EXT": [ {"vuid": "<id>", "text": "<VU txt>"}]}, |
| # "VK_KHX_external_memory" & "VK_KHX_device_group" - extension case (vs. "core") |
| for top_level in sorted(self.json_data): |
| if "validation" == top_level: |
| for api in sorted(self.json_data[top_level]): |
| for ext in sorted(self.json_data[top_level][api]): |
| for vu_txt_dict in self.json_data[top_level][api][ext]: |
| print ("Looking at dict for api:ext entry %s:%s" % (api, ext)) |
| vuid = vu_txt_dict['vuid'] |
| vutxt = vu_txt_dict['text'] |
| # strip asciidoc xref from vu text |
| vutxt = re.sub('&amp;lt;&amp;lt;([^&]*,\\s*|)(.*?)&amp;gt;&amp;gt;', '\\2', vutxt) |
| #print ("%s:%s:%s:%s" % (api, ext, vuid, vutxt)) |
| #print ("VUTXT orig:%s" % (vutxt)) |
| just_txt = BeautifulSoup(vutxt, 'html.parser') |
| #print ("VUTXT only:%s" % (just_txt.get_text())) |
| num_vuid = vuid_mapping.convertVUID(vuid) |
| self.json_db[vuid] = {} |
| self.json_db[vuid]['ext'] = ext |
| self.json_db[vuid]['number_vuid'] = num_vuid |
| self.json_db[vuid]['struct_func'] = api |
| just_txt = just_txt.get_text().strip() |
| unicode_map = { |
| u"\u2019" : "'", |
| u"\u201c" : "\"", |
| u"\u201d" : "\"", |
| u"\u2192" : "->", |
| } |
| for um in unicode_map: |
| just_txt = just_txt.replace(um, unicode_map[um]) |
| self.json_db[vuid]['vu_txt'] = just_txt.replace("\\", "") |
| print ("Spec vu txt:%s" % (self.json_db[vuid]['vu_txt'])) |
| #sys.exit() |
| |
| def compareJSON(self): |
| """Compare parsed json file with existing data read in from DB file""" |
| # update database for all json vuids |
| for vuid, vuid_json_data in self.json_db.items(): |
| # convert vuid to error enum |
| error_enum = "%s%s" % (validation_error_enum_name, get8digithex(vuid_json_data['number_vuid'])) |
| # create database entry if one doesn't exist |
| if error_enum not in self.error_db_dict: |
| self.error_db_dict[error_enum] = {'check_implemented': 'N', |
| 'testname': 'None', |
| 'note': ''} |
| vuid_db_data = self.error_db_dict[error_enum] |
| # check if vuid is implemented but changed extension scope |
| if vuid_db_data['check_implemented'] == 'Y' and \ |
| vuid_db_data['ext'] != vuid_json_data['ext']: |
| # should not occur often, currently a hard error to force corrective action |
| print('ERROR: {}/{} is currently implemented and changed extension scope from "{}" to "{}"'.format( |
| vuid, error_enum, vuid_db_data['ext'], vuid_json_data['ext'])) |
| exit(1) |
| # update database entry with data from json file |
| if 'core' == vuid_json_data['ext'] or '!' in vuid_json_data['ext']: |
| spec_link = "%s#%s" % (core_url, vuid) |
| else: |
| spec_link = "%s#%s" % (ext_url, vuid) |
| vuid_db_data['api'] = vuid_json_data['struct_func'] |
| vuid_db_data['vuid_string'] = vuid |
| vuid_db_data['error_msg'] = "%s'%s' (%s)" % (error_msg_prefix, vuid_json_data['vu_txt'], spec_link) |
| vuid_db_data['ext'] = vuid_json_data['ext'] |
| last_segment = vuid.split("-")[-1] |
| vuid_db_data['implicit'] = not last_segment.isdigit() |
| |
| # remove missing vuids from database |
| for enum in list(self.error_db_dict): |
| vuid = self.error_db_dict[enum]['vuid_string'] |
| if vuid not in self.json_db: |
| print ("WARN: Couldn't find vuid_string in json db:%s" % (vuid)) |
| del self.error_db_dict[enum] |
| |
| def genHeader(self, header_file): |
| """Generate a header file based on the contents of a parsed spec""" |
| print ("Generating header %s..." % (header_file)) |
| file_contents = [] |
| file_contents.append(self.copyright) |
| file_contents.append('\n#pragma once') |
| file_contents.append('\n// Disable auto-formatting for generated file') |
| file_contents.append('// clang-format off') |
| file_contents.append('\n#include <string>') |
| file_contents.append('#include <unordered_map>') |
| file_contents.append('\n// enum values for unique validation error codes') |
| file_contents.append('// Corresponding validation error message for each enum is given in the mapping table below') |
| file_contents.append('// When a given error occurs, these enum values should be passed to the as the messageCode') |
| file_contents.append('// parameter to the PFN_vkDebugReportCallbackEXT function') |
| enum_decl = ['enum UNIQUE_VALIDATION_ERROR_CODE {\n VALIDATION_ERROR_UNDEFINED = -1,'] |
| vuid_int_to_error_map = ['static std::unordered_map<int, char const *const> validation_error_map{'] |
| vuid_string_to_error_map = ['static std::unordered_map<std::string, int> validation_error_text_map{'] |
| enum_value = 0 |
| max_enum_val = 0 |
| for enum in sorted(self.error_db_dict): |
| enum_decl.append(' %s = 0x%s,' % (enum, enum[-8:])) |
| vuid_int_to_error_map.append(' {%s, "%s"},' % (enum, self.error_db_dict[enum]['error_msg'].replace('"', '\\"'))) |
| vuid_str = self.error_db_dict[enum]['vuid_string'] |
| vuid_string_to_error_map.append(' {"%s", %s},' % (vuid_str, enum)) |
| max_enum_val = max(max_enum_val, enum_value) |
| enum_decl.append(' %sMAX_ENUM = %d,' % (validation_error_enum_name, max_enum_val + 1)) |
| enum_decl.append('};') |
| vuid_int_to_error_map.append('};\n') |
| vuid_string_to_error_map.append('};\n') |
| file_contents.extend(enum_decl) |
| file_contents.append('\n// Mapping from unique validation error enum to the corresponding spec text') |
| file_contents.extend(vuid_int_to_error_map) |
| file_contents.append('\n// Mapping from spec validation error text string to unique validation error enum') |
| file_contents.extend(vuid_string_to_error_map) |
| #print ("File contents: %s" % (file_contents)) |
| with open(header_file, "w") as outfile: |
| outfile.write("\n".join(file_contents)) |
| def genDB(self, db_file): |
| """Generate a database of check_enum, check_coded?, testname, API, VUID_string, core|ext, error_string, notes""" |
| db_lines = [] |
| # Write header for database file |
| db_lines.append("# This is a database file with validation error check information") |
| db_lines.append("# Comments are denoted with '#' char") |
| db_lines.append("# The format of the lines is:") |
| db_lines.append("# <error_enum>%s<check_implemented>%s<testname>%s<api>%s<vuid_string>%s<core|ext>%s<errormsg>%s<note>" % (self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter)) |
| db_lines.append("# error_enum: Unique error enum for this check of format %s<uniqueid>" % validation_error_enum_name) |
| db_lines.append("# check_implemented: 'Y' if check has been implemented in layers, or 'N' for not implemented") |
| db_lines.append("# testname: Name of validation test for this check, 'Unknown' for unknown, 'None' if not implemented, or 'NotTestable' if cannot be implemented") |
| db_lines.append("# api: Vulkan API function that this check is related to") |
| db_lines.append("# vuid_string: Unique string to identify this check") |
| db_lines.append("# core|ext: Either 'core' for core spec or some extension string that indicates the extension required for this VU to be relevant") |
| db_lines.append("# errormsg: The unique error message for this check that includes spec language and link") |
| db_lines.append("# note: Free txt field with any custom notes related to the check in question") |
| for enum in sorted(self.error_db_dict): |
| print ("Gen DB for enum %s" % (enum)) |
| implicit = self.error_db_dict[enum]['implicit'] |
| implemented = self.error_db_dict[enum]['check_implemented'] |
| testname = self.error_db_dict[enum]['testname'] |
| note = self.error_db_dict[enum]['note'] |
| core_ext = self.error_db_dict[enum]['ext'] |
| self.error_db_dict[enum]['vuid_string'] = self.error_db_dict[enum]['vuid_string'] |
| if implicit and 'implicit' not in note: # add implicit note |
| if '' != note: |
| note = "implicit, %s" % (note) |
| else: |
| note = "implicit" |
| db_lines.append("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s" % (enum, self.delimiter, implemented, self.delimiter, testname, self.delimiter, self.error_db_dict[enum]['api'], self.delimiter, self.error_db_dict[enum]['vuid_string'], self.delimiter, core_ext, self.delimiter, self.error_db_dict[enum]['error_msg'], self.delimiter, note)) |
| db_lines.append("\n") # newline at end of file |
| print ("Generating database file %s" % (db_file)) |
| with open(db_file, "w") as outfile: |
| outfile.write("\n".join(db_lines)) |
| def readDB(self, db_file): |
| """Read a db file into a dict, refer to genDB function above for format of each line""" |
| with open(db_file, "r", encoding='utf-8') as infile: |
| for line in infile: |
| line = line.strip() |
| if line.startswith('#') or '' == line: |
| continue |
| db_line = line.split(self.delimiter) |
| if len(db_line) != 8: |
| print ("ERROR: Bad database line doesn't have 8 elements: %s" % (line)) |
| error_enum = db_line[0] |
| implemented = db_line[1] |
| testname = db_line[2] |
| api = db_line[3] |
| vuid_str = db_line[4] |
| core_ext = db_line[5] |
| error_str = db_line[6] |
| note = db_line[7] |
| # Also read complete database contents into our class var for later use |
| self.error_db_dict[error_enum] = {} |
| self.error_db_dict[error_enum]['check_implemented'] = implemented |
| self.error_db_dict[error_enum]['testname'] = testname |
| self.error_db_dict[error_enum]['api'] = api |
| self.error_db_dict[error_enum]['vuid_string'] = vuid_str |
| self.error_db_dict[error_enum]['ext'] = core_ext |
| self.error_db_dict[error_enum]['error_msg'] = error_str |
| self.error_db_dict[error_enum]['note'] = note |
| implicit = False |
| last_segment = vuid_str.split("-")[-1] |
| if last_segment in vuid_mapping.implicit_type_map: |
| implicit = True |
| elif not last_segment.isdigit(): # Explicit ids should only have digits in last segment |
| print ("ERROR: Found last segment of val error ID that isn't in implicit map and doesn't have numbers in last segment: %s" % (last_segment)) |
| sys.exit() |
| self.error_db_dict[error_enum]['implicit'] = implicit |
| if __name__ == "__main__": |
| i = 1 |
| use_online = True # Attempt to grab spec from online by default |
| while (i < len(sys.argv)): |
| arg = sys.argv[i] |
| i = i + 1 |
| if (arg == '-json-file'): |
| json_filename = sys.argv[i] |
| i = i + 1 |
| elif (arg == '-json-compare'): |
| json_compare = True |
| elif (arg == '-out'): |
| out_filename = sys.argv[i] |
| i = i + 1 |
| elif (arg == '-gendb'): |
| gen_db = True |
| # Set filename if supplied, else use default |
| if i < len(sys.argv) and not sys.argv[i].startswith('-'): |
| db_filename = sys.argv[i] |
| i = i + 1 |
| elif (arg == '-update'): |
| json_compare = True |
| gen_db = True |
| elif (arg in ['-help', '-h']): |
| printHelp() |
| sys.exit() |
| spec = Specification() |
| spec.readJSON() |
| spec.parseJSON() |
| if (json_compare): |
| # Read in current spec info from db file |
| (orig_err_msg_dict) = spec.readDB(db_filename) |
| spec.compareJSON() |
| print ("Found %d missing db entries in json db" % (spec.json_missing)) |
| print ("Found %d duplicate json entries" % (spec.duplicate_json_key_count)) |
| spec.genDB(db_filename) |
| if (gen_db): |
| spec.genDB(db_filename) |
| print ("Writing out file (-out) to '%s'" % (out_filename)) |
| spec.genHeader(out_filename) |
