| #!/usr/bin/python3 |
| # |
| # Copyright 2016-2021 The Khronos Group Inc. |
| # |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| """Used for automatic reflow of spec sources to satisfy the agreed layout to |
| minimize git churn. Most of the logic has to do with detecting asciidoc |
| markup or block types that *shouldn't* be reflowed (tables, code) and |
| ignoring them. It's very likely there are many asciidoc constructs not yet |
| accounted for in the script, our usage of asciidoc markup is intentionally |
| somewhat limited. |
| |
| Also used to insert identifying tags on explicit Valid Usage statements. |
| |
| Usage: `reflow.py [-noflow] [-tagvu] [-nextvu #] [-overwrite] [-out dir] [-suffix str] files` |
| |
| - `-noflow` acts as a passthrough, instead of reflowing text. Other |
| processing may occur. |
| - `-tagvu` generates explicit VUID tag for Valid Usage statements which |
| don't already have them. |
| - `-nextvu #` starts VUID tag generation at the specified # instead of |
| the value wired into the `reflow.py` script. |
| - `-overwrite` updates in place (can be risky, make sure there are backups) |
| - `-check FAIL|WARN` runs some simple sanity checks on markup. If the checks |
| fail and the WARN option is given, the script will simply print a warning |
| message. If the checks fail and the FAIL option is given, the script will |
| exit with an error code. FAIL is for use with continuous integration |
| scripts enforcing the checks. |
| - `-out` specifies directory to create output file in, default 'out' |
| - `-suffix` specifies suffix to add to output files, default '' |
| - `files` are asciidoc source files from the spec to reflow. |
| """ |
| # For error and file-loading interfaces only |
| import argparse |
| import os |
| import re |
| import sys |
| from reflib import loadFile, logDiag, logWarn, logErr, setLogFile, getBranch |
| |
| # Vulkan-specific - will consolidate into scripts/ like OpenXR soon |
| sys.path.insert(0, 'xml') |
| |
| from vkconventions import VulkanConventions as APIConventions |
| conventions = APIConventions() |
| |
| # Markup that always ends a paragraph |
| # empty line or whitespace |
| # [block options] |
| # [[anchor]] |
| # // comment |
| # <<<< page break |
| # :attribute-setting |
| # macro-directive::terms |
| # + standalone list item continuation |
| # label:: labelled list - label must be standalone |
| endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|:.*|[a-z]+::.*|\+|.*::)$') |
| |
| # Special case of markup ending a paragraph, used to track the current |
| # command/structure. This allows for either OpenXR or Vulkan API path |
| # conventions. Nominally it should use the file suffix defined by the API |
| # conventions (conventions.file_suffix), except that XR uses '.txt' for |
| # generated API include files, not '.adoc' like its other includes. |
| includePat = re.compile( |
| r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') |
| |
| # Find the first pname: or code: pattern in a Valid Usage statement |
| pnamePat = re.compile(r'pname:(?P<param>\{?\w+\}?)') |
| codePat = re.compile(r'code:(?P<param>\w+)') |
| |
| # Markup that's OK in a contiguous paragraph but otherwise passed through |
| # .anything (except .., which indicates a literal block) |
| # === Section Titles |
| endParaContinue = re.compile(r'^(\.[^.].*|=+ .*)$') |
| |
| # Markup for block delimiters whose contents *should* be reformatted |
| # -- (exactly two) (open block) |
| # **** (4 or more) (sidebar block - why do we have these?!) |
| # ==== (4 or more) (example block) |
| # ____ (4 or more) (quote block) |
| blockReflow = re.compile(r'^(--|[*=_]{4,})$') |
| |
| # Fake block delimiters for "common" VU statements |
| blockCommonReflow = '// Common Valid Usage\n' |
| |
| # Markup for block delimiters whose contents should *not* be reformatted |
| # |=== (3 or more) (table) |
| # ++++ (4 or more) (passthrough block) |
| # .... (4 or more) (literal block) |
| # //// (4 or more) (comment block) |
| # ---- (4 or more) (listing block) |
| # ``` (3 or more) (listing block) |
| # **** (4 or more) (sidebar block) |
| blockPassthrough = re.compile(r'^(\|={3,}|[`]{3}|[\-+./~]{4,})$') |
| |
| # Markup for introducing lists (hanging paragraphs) |
| # * bullet |
| # ** bullet |
| # -- bullet |
| # . bullet |
| # :: bullet (no longer supported by asciidoctor 2) |
| # {empty}:: bullet |
| # 1. list item |
| beginBullet = re.compile(r'^ *([*\-.]+|\{empty\}::|::|[0-9]+[.]) ') |
| |
| # Start of an asciidoctor conditional |
| # ifdef:: |
| # ifndef:: |
| conditionalStart = re.compile(r'^(ifdef|ifndef)::') |
| |
| # Text that (may) not end sentences |
| |
| # A single letter followed by a period, typically a middle initial. |
| endInitial = re.compile(r'^[A-Z]\.$') |
| # An abbreviation, which doesn't (usually) end a line. |
| endAbbrev = re.compile(r'(e\.g|i\.e|c\.f|vs)\.$', re.IGNORECASE) |
| |
| class ReflowState: |
| """State machine for reflowing. |
| |
| Represents the state of the reflow operation""" |
| def __init__(self, |
| filename, |
| margin = 76, |
| file = sys.stdout, |
| breakPeriod = True, |
| reflow = True, |
| nextvu = None, |
| maxvu = None): |
| |
| self.blockStack = [ None ] |
| """The last element is a line with the asciidoc block delimiter that's currently in effect, |
| such as '--', '----', '****', '======', or '+++++++++'. |
| This affects whether or not the block contents should be formatted.""" |
| |
| self.reflowStack = [ True ] |
| """The last element is True or False if the current blockStack contents |
| should be reflowed.""" |
| self.vuStack = [ False ] |
| """the last element is True or False if the current blockStack contents |
| are an explicit Valid Usage block.""" |
| |
| self.margin = margin |
| """margin to reflow text to.""" |
| |
| self.para = [] |
| """list of lines in the paragraph being accumulated. |
| When this is non-empty, there is a current paragraph.""" |
| |
| self.lastTitle = False |
| """true if the previous line was a document title line |
| (e.g. :leveloffset: 0 - no attempt to track changes to this is made).""" |
| |
| self.leadIndent = 0 |
| """indent level (in spaces) of the first line of a paragraph.""" |
| |
| self.hangIndent = 0 |
| """indent level of the remaining lines of a paragraph.""" |
| |
| self.file = file |
| """file handle to write to.""" |
| |
| self.filename = filename |
| """base name of file being read from.""" |
| |
| self.lineNumber = 0 |
| """line number being read from the input file.""" |
| |
| self.breakPeriod = breakPeriod |
| """True if justification should break to a new line after the end of a sentence.""" |
| |
| self.breakInitial = True |
| """True if justification should break to a new line after |
| something that appears to be an initial in someone's name. **TBD**""" |
| |
| self.reflow = reflow |
| """True if text should be reflowed, False to pass through unchanged.""" |
| |
| self.vuPrefix = 'VUID' |
| """Prefix of generated Valid Usage tags""" |
| |
| self.vuFormat = '{0}-{1}-{2}-{3:0>5d}' |
| """Format string for generating Valid Usage tags. |
| First argument is vuPrefix, second is command/struct name, third is parameter name, fourth is the tag number.""" |
| |
| self.nextvu = nextvu |
| """Integer to start tagging un-numbered Valid Usage statements with, |
| or None if no tagging should be done.""" |
| |
| self.maxvu = maxvu |
| """Maximum tag to use for Valid Usage statements, or None if no |
| tagging should be done.""" |
| |
| self.defaultApiName = '{refpage}' |
| self.apiName = self.defaultApiName |
| """String name of a Vulkan structure or command for VUID tag |
| generation, or {refpage} if one hasn't been included in this file |
| yet.""" |
| |
| def incrLineNumber(self): |
| self.lineNumber = self.lineNumber + 1 |
| |
| def printLines(self, lines): |
| """Print an array of lines with newlines already present""" |
| if len(lines) > 0: |
| logDiag(':: printLines:', len(lines), 'lines: ', lines[0], end='') |
| |
| if self.file is not None: |
| for line in lines: |
| print(line, file=self.file, end='') |
| |
| def endSentence(self, word): |
| """Return True if word ends with a sentence-period, False otherwise. |
| |
| Allows for contraction cases which won't end a line: |
| |
| - A single letter (if breakInitial is True) |
| - Abbreviations: 'c.f.', 'e.g.', 'i.e.' (or mixed-case versions)""" |
| if (word[-1:] != '.' or |
| endAbbrev.search(word) or |
| (self.breakInitial and endInitial.match(word))): |
| return False |
| |
| return True |
| |
| def vuidAnchor(self, word): |
| """Return True if word is a Valid Usage ID Tag anchor.""" |
| return (word[0:7] == '[[VUID-') |
| |
| def isOpenBlockDelimiter(self, line): |
| """Returns True if line is an open block delimiter.""" |
| return line[0:2] == '--' |
| |
| def reflowPara(self): |
| """Reflow the current paragraph, respecting the paragraph lead and |
| hanging indentation levels. |
| |
| The algorithm also respects trailing '+' signs that indicate embedded newlines, |
| and will not reflow a very long word immediately after a bullet point. |
| |
| Just return the paragraph unchanged if the -noflow argument was |
| given.""" |
| if not self.reflow: |
| return self.para |
| |
| logDiag('reflowPara lead indent = ', self.leadIndent, |
| 'hangIndent =', self.hangIndent, |
| 'para:', self.para[0], end='') |
| |
| # Total words processed (we care about the *first* word vs. others) |
| wordCount = 0 |
| |
| # Tracks the *previous* word processed. It must not be empty. |
| prevWord = ' ' |
| |
| # Track the previous line and paragraph being indented, if any |
| outLine = None |
| outPara = [] |
| |
| for line in self.para: |
| line = line.rstrip() |
| words = line.split() |
| |
| # logDiag('reflowPara: input line =', line) |
| numWords = len(words) - 1 |
| |
| for i in range(0, numWords + 1): |
| word = words[i] |
| wordLen = len(word) |
| wordCount += 1 |
| |
| endEscape = False |
| if i == numWords and word == '+': |
| # Trailing ' +' must stay on the same line |
| endEscape = word |
| # logDiag('reflowPara last word of line =', word, 'prevWord =', prevWord, 'endEscape =', endEscape) |
| else: |
| pass |
| # logDiag('reflowPara wordCount =', wordCount, 'word =', word, 'prevWord =', prevWord) |
| |
| if wordCount == 1: |
| # The first word of the paragraph is treated specially. |
| # The loop logic becomes trickier if all this code is |
| # done prior to looping over lines and words, so all the |
| # setup logic is done here. |
| |
| outPara = [] |
| outLine = ''.ljust(self.leadIndent) + word |
| outLineLen = self.leadIndent + wordLen |
| |
| # If the paragraph begins with a bullet point, generate |
| # a hanging indent level if there isn't one already. |
| if beginBullet.match(self.para[0]): |
| bulletPoint = True |
| if len(self.para) > 1: |
| logDiag('reflowPara first line matches bullet point', |
| 'but indent already hanging @ input line', |
| self.lineNumber) |
| else: |
| logDiag('reflowPara first line matches bullet point -' |
| 'single line, assuming hangIndent @ input line', |
| self.lineNumber) |
| self.hangIndent = outLineLen + 1 |
| else: |
| bulletPoint = False |
| else: |
| # Possible actions to take with this word |
| # |
| # addWord - add word to current line |
| # closeLine - append line and start a new (null) one |
| # startLine - add word to a new line |
| |
| # Default behavior if all the tests below fail is to add |
| # this word to the current line, and keep accumulating |
| # that line. |
| (addWord, closeLine, startLine) = (True, False, False) |
| |
| # How long would this line be if the word were added? |
| newLen = outLineLen + 1 + wordLen |
| |
| # Are we on the first word following a bullet point? |
| firstBullet = (wordCount == 2 and bulletPoint) |
| |
| if endEscape: |
| # If the new word ends the input line with ' +', |
| # add it to the current line. |
| |
| (addWord, closeLine, startLine) = (True, True, False) |
| elif self.vuidAnchor(word): |
| # If the new word is a Valid Usage anchor, break the |
| # line afterwards. Note that this should only happen |
| # immediately after a bullet point, but we don't |
| # currently check for this. |
| (addWord, closeLine, startLine) = (True, True, False) |
| elif newLen > self.margin: |
| if firstBullet: |
| # If the word follows a bullet point, add it to |
| # the current line no matter its length. |
| |
| (addWord, closeLine, startLine) = (True, True, False) |
| elif beginBullet.match(word + ' '): |
| # If the word *is* a bullet point, add it to |
| # the current line no matter its length. |
| # This avoids an innocent inline '-' or '*' |
| # turning into a bogus bullet point. |
| |
| (addWord, closeLine, startLine) = (True, True, False) |
| else: |
| # The word overflows, so add it to a new line. |
| |
| (addWord, closeLine, startLine) = (False, True, True) |
| elif (self.breakPeriod and |
| (wordCount > 2 or not firstBullet) and |
| self.endSentence(prevWord)): |
| # If the previous word ends a sentence and |
| # breakPeriod is set, start a new line. |
| # The complicated logic allows for leading bullet |
| # points which are periods (implicitly numbered lists). |
| # @@@ But not yet for explicitly numbered lists. |
| |
| (addWord, closeLine, startLine) = (False, True, True) |
| |
| # Add a word to the current line |
| if addWord: |
| if outLine: |
| outLine += ' ' + word |
| outLineLen = newLen |
| else: |
| # Fall through to startLine case if there's no |
| # current line yet. |
| startLine = True |
| |
| # Add current line to the output paragraph. Force |
| # starting a new line, although we don't yet know if it |
| # will ever have contents. |
| if closeLine: |
| if outLine: |
| outPara.append(outLine + '\n') |
| outLine = None |
| |
| # Start a new line and add a word to it |
| if startLine: |
| outLine = ''.ljust(self.hangIndent) + word |
| outLineLen = self.hangIndent + wordLen |
| |
| # Track the previous word, for use in breaking at end of |
| # a sentence |
| prevWord = word |
| |
| # Add this line to the output paragraph. |
| if outLine: |
| outPara.append(outLine + '\n') |
| |
| return outPara |
| |
| def emitPara(self): |
| """Emit a paragraph, possibly reflowing it depending on the block context. |
| |
| Resets the paragraph accumulator.""" |
| if self.para != []: |
| if self.vuStack[-1] and self.nextvu is not None: |
| # If: |
| # - this paragraph is in a Valid Usage block, |
| # - VUID tags are being assigned, |
| # Try to assign VUIDs |
| |
| if nestedVuPat.search(self.para[0]): |
| # Check for nested bullet points. These should not be |
| # assigned VUIDs, nor present at all, because they break |
| # the VU extractor. |
| logWarn(self.filename + ': Invalid nested bullet point in VU block:', self.para[0]) |
| elif self.vuPrefix not in self.para[0]: |
| # If: |
| # - a tag is not already present, and |
| # - the paragraph is a properly marked-up list item |
| # Then add a VUID tag starting with the next free ID. |
| |
| # Split the first line after the bullet point |
| matches = vuPat.search(self.para[0]) |
| if matches is not None: |
| logDiag('findRefs: Matched vuPat on line:', self.para[0], end='') |
| head = matches.group('head') |
| tail = matches.group('tail') |
| |
| # Use the first pname: or code: tag in the paragraph as |
| # the parameter name in the VUID tag. This won't always |
| # be correct, but should be highly reliable. |
| for vuLine in self.para: |
| matches = pnamePat.search(vuLine) |
| if matches is not None: |
| break |
| matches = codePat.search(vuLine) |
| if matches is not None: |
| break |
| |
| if matches is not None: |
| paramName = matches.group('param') |
| else: |
| paramName = 'None' |
| logWarn(self.filename, |
| 'No param name found for VUID tag on line:', |
| self.para[0]) |
| |
| newline = (head + ' [[' + |
| self.vuFormat.format(self.vuPrefix, |
| self.apiName, |
| paramName, |
| self.nextvu) + ']] ' + tail) |
| |
| logDiag('Assigning', self.vuPrefix, self.apiName, self.nextvu, |
| ' on line:', self.para[0], '->', newline, 'END') |
| |
| # Don't actually assign the VUID unless it's in the reserved range |
| if self.nextvu <= self.maxvu: |
| if self.nextvu == self.maxvu: |
| logWarn('Skipping VUID assignment, no more VUIDs available') |
| self.para[0] = newline |
| self.nextvu = self.nextvu + 1 |
| # else: |
| # There are only a few cases of this, and they're all |
| # legitimate. Leave detecting this case to another tool |
| # or hand inspection. |
| # logWarn(self.filename + ': Unexpected non-bullet item in VU block (harmless if following an ifdef):', |
| # self.para[0]) |
| |
| if self.reflowStack[-1]: |
| self.printLines(self.reflowPara()) |
| else: |
| self.printLines(self.para) |
| |
| # Reset the paragraph, including its indentation level |
| self.para = [] |
| self.leadIndent = 0 |
| self.hangIndent = 0 |
| |
| def endPara(self, line): |
| """'line' ends a paragraph and should itself be emitted. |
| line may be None to indicate EOF or other exception.""" |
| logDiag('endPara line', self.lineNumber, ': emitting paragraph') |
| |
| # Emit current paragraph, this line, and reset tracker |
| self.emitPara() |
| |
| if line: |
| self.printLines( [ line ] ) |
| |
| def endParaContinue(self, line): |
| """'line' ends a paragraph (unless there's already a paragraph being |
| accumulated, e.g. len(para) > 0 - currently not implemented)""" |
| self.endPara(line) |
| |
| def endBlock(self, line, reflow = False, vuBlock = False): |
| """'line' begins or ends a block. |
| |
| If beginning a block, tag whether or not to reflow the contents. |
| |
| vuBlock is True if the previous line indicates this is a Valid Usage block.""" |
| self.endPara(line) |
| |
| if self.blockStack[-1] == line: |
| logDiag('endBlock line', self.lineNumber, |
| ': popping block end depth:', len(self.blockStack), |
| ':', line, end='') |
| |
| # Reset apiName at the end of an open block. |
| # Open blocks cannot be nested (at present), so this is safe. |
| if self.isOpenBlockDelimiter(line): |
| logDiag('reset apiName to empty at line', self.lineNumber) |
| self.apiName = self.defaultApiName |
| else: |
| logDiag('NOT resetting apiName to default at line', self.lineNumber) |
| |
| self.blockStack.pop() |
| self.reflowStack.pop() |
| self.vuStack.pop() |
| else: |
| # Start a block |
| self.blockStack.append(line) |
| self.reflowStack.append(reflow) |
| self.vuStack.append(vuBlock) |
| |
| logDiag('endBlock reflow =', reflow, ' line', self.lineNumber, |
| ': pushing block start depth', len(self.blockStack), |
| ':', line, end='') |
| |
| def endParaBlockReflow(self, line, vuBlock): |
| """'line' begins or ends a block. The paragraphs in the block *should* be |
| reformatted (e.g. a NOTE).""" |
| self.endBlock(line, reflow = True, vuBlock = vuBlock) |
| |
| def endParaBlockPassthrough(self, line): |
| """'line' begins or ends a block. The paragraphs in the block should |
| *not* be reformatted (e.g. a code listing).""" |
| self.endBlock(line, reflow = False) |
| |
| def addLine(self, line): |
| """'line' starts or continues a paragraph. |
| |
| Paragraphs may have "hanging indent", e.g. |
| |
| ``` |
| * Bullet point... |
| ... continued |
| ``` |
| |
| In this case, when the higher indentation level ends, so does the |
| paragraph.""" |
| logDiag('addLine line', self.lineNumber, ':', line, end='') |
| |
| # See https://stackoverflow.com/questions/13648813/what-is-the-pythonic-way-to-count-the-leading-spaces-in-a-string |
| indent = len(line) - len(line.lstrip()) |
| |
| # A hanging paragraph ends due to a less-indented line. |
| if self.para != [] and indent < self.hangIndent: |
| logDiag('addLine: line reduces indentation, emit paragraph') |
| self.emitPara() |
| |
| # A bullet point (or something that looks like one) always ends the |
| # current paragraph. |
| if beginBullet.match(line): |
| logDiag('addLine: line matches beginBullet, emit paragraph') |
| self.emitPara() |
| |
| if self.para == []: |
| # Begin a new paragraph |
| self.para = [ line ] |
| self.leadIndent = indent |
| self.hangIndent = indent |
| else: |
| # Add a line to a paragraph. Increase the hanging indentation |
| # level - once. |
| if self.hangIndent == self.leadIndent: |
| self.hangIndent = indent |
| self.para.append(line) |
| |
| def apiMatch(oldname, newname): |
| """Returns whether oldname and newname match, up to an API suffix. |
| This should use the API map instead of this heuristic, since aliases |
| like VkPhysicalDeviceVariablePointerFeatures -> |
| VkPhysicalDeviceVariablePointersFeatures are not recognized.""" |
| upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| return oldname.rstrip(upper) == newname.rstrip(upper) |
| |
| def reflowFile(filename, args): |
| logDiag('reflow: filename', filename) |
| |
| lines = loadFile(filename) |
| if lines is None: |
| return |
| |
| # Output file handle and reflow object for this file. There are no race |
| # conditions on overwriting the input, but it's not recommended unless |
| # you have backing store such as git. |
| |
| if args.overwrite: |
| outFilename = filename |
| else: |
| outFilename = args.outDir + '/' + os.path.basename(filename) + args.suffix |
| |
| if args.nowrite: |
| fp = None |
| else: |
| try: |
| fp = open(outFilename, 'w', encoding='utf8') |
| except: |
| logWarn('Cannot open output file', outFilename, ':', sys.exc_info()[0]) |
| return |
| |
| state = ReflowState(filename, |
| margin = args.margin, |
| file = fp, |
| reflow = not args.noflow, |
| nextvu = args.nextvu, |
| maxvu = args.maxvu) |
| |
| for line in lines: |
| state.incrLineNumber() |
| |
| # Is this a title line (leading '= ' followed by text)? |
| thisTitle = False |
| |
| matches = vuidPat.search(line) |
| if matches is not None: |
| # If we found a VUID pattern, add the (filename,line) it was |
| # found at to a list for that VUID, to find duplicates. |
| vuid = matches.group('vuid') |
| if vuid not in args.vuidDict: |
| args.vuidDict[vuid] = [] |
| args.vuidDict[vuid].append([filename, line]) |
| |
| # The logic here is broken. If we're in a non-reflowable block and |
| # this line *doesn't* end the block, it should always be |
| # accumulated. |
| |
| # Test for a blockCommonReflow delimiter comment first, to avoid |
| # treating it solely as a end-Paragraph marker comment. |
| if line == blockCommonReflow: |
| # Starting or ending a pseudo-block for "common" VU statements. |
| state.endParaBlockReflow(line, vuBlock = True) |
| |
| elif blockReflow.match(line): |
| # Starting or ending a block whose contents may be reflowed. |
| # Blocks cannot be nested. |
| |
| # Is this is an explicit Valid Usage block? |
| vuBlock = (state.lineNumber > 1 and |
| lines[state.lineNumber-2] == '.Valid Usage\n') |
| |
| state.endParaBlockReflow(line, vuBlock) |
| |
| elif endPara.match(line): |
| # Ending a paragraph. Emit the current paragraph, if any, and |
| # prepare to begin a new paragraph. |
| |
| state.endPara(line) |
| |
| # If this is an include:: line starting the definition of a |
| # structure or command, track that for use in VUID generation. |
| |
| matches = includePat.search(line) |
| if matches is not None: |
| generated_type = matches.group('generated_type') |
| include_type = matches.group('category') |
| if generated_type == 'api' and include_type in ('protos', 'structs', 'funcpointers'): |
| apiName = matches.group('entity_name') |
| if state.apiName != state.defaultApiName: |
| # This happens when there are multiple API include |
| # lines in a single block. The style guideline is to |
| # always place the API which others are promoted to |
| # first. In virtually all cases, the promoted API |
| # will differ solely in the vendor suffix (or |
| # absence of it), which is benign. |
| if not apiMatch(state.apiName, apiName): |
| logDiag(f'Promoted API name mismatch at line {state.lineNumber}: {apiName} does not match state.apiName (this is OK if it is just a spelling alias)') |
| else: |
| state.apiName = apiName |
| |
| elif endParaContinue.match(line): |
| # For now, always just end the paragraph. |
| # Could check see if len(para) > 0 to accumulate. |
| |
| state.endParaContinue(line) |
| |
| # If it's a title line, track that |
| if line[0:2] == '= ': |
| thisTitle = True |
| |
| elif blockPassthrough.match(line): |
| # Starting or ending a block whose contents must not be reflowed. |
| # These are tables, etc. Blocks cannot be nested. |
| |
| state.endParaBlockPassthrough(line) |
| elif state.lastTitle: |
| # The previous line was a document title line. This line |
| # is the author / credits line and must not be reflowed. |
| |
| state.endPara(line) |
| else: |
| # Just accumulate a line to the current paragraph. Watch out for |
| # hanging indents / bullet-points and track that indent level. |
| |
| state.addLine(line) |
| |
| # This test looks for disallowed conditionals inside Valid Usage |
| # blocks, by checking if (a) this line does not start a new VU |
| # (bullet point) and (b) the previous line starts an asciidoctor |
| # conditional (ifdef:: or ifndef::). |
| |
| if (args.check |
| and state.vuStack[-1] |
| and not beginBullet.match(line) |
| and conditionalStart.match(lines[state.lineNumber-2])): |
| |
| logWarn('Detected embedded Valid Usage conditional: {}:{}'.format( |
| filename, state.lineNumber - 1)) |
| # Keep track of warning check count |
| args.warnCount = args.warnCount + 1 |
| |
| state.lastTitle = thisTitle |
| |
| # Cleanup at end of file |
| state.endPara(None) |
| |
| # Check for sensible block nesting |
| if len(state.blockStack) > 1: |
| logWarn('file', filename, |
| 'mismatched asciidoc block delimiters at EOF:', |
| state.blockStack[-1]) |
| |
| if fp is not None: |
| fp.close() |
| |
| # Update the 'nextvu' value |
| if args.nextvu != state.nextvu: |
| logWarn('Updated nextvu to', state.nextvu, 'after file', filename) |
| args.nextvu = state.nextvu |
| |
| def reflowAllAdocFiles(folder_to_reflow, args): |
| for root, subdirs, files in os.walk(folder_to_reflow): |
| for file in files: |
| if file.endswith(conventions.file_suffix): |
| file_path = os.path.join(root, file) |
| reflowFile(file_path, args) |
| for subdir in subdirs: |
| sub_folder = os.path.join(root, subdir) |
| print('Sub-folder = %s' % sub_folder) |
| if subdir.lower() not in conventions.spec_no_reflow_dirs: |
| print(' Parsing = %s' % sub_folder) |
| reflowAllAdocFiles(sub_folder, args) |
| else: |
| print(' Skipping = %s' % sub_folder) |
| |
| # Patterns used to recognize interesting lines in an asciidoc source file. |
| # These patterns are only compiled once. |
| |
| # Explicit Valid Usage list item with one or more leading asterisks |
| # The re.DOTALL is needed to prevent vuPat.search() from stripping |
| # the trailing newline. |
| vuPat = re.compile(r'^(?P<head> [*]+)( *)(?P<tail>.*)', re.DOTALL) |
| |
| # VUID with the numeric portion captured in the match object |
| vuidPat = re.compile(r'VUID-[^-]+-[^-]+-(?P<vuid>[0-9]+)') |
| |
| # Pattern matching leading nested bullet points |
| global nestedVuPat |
| nestedVuPat = re.compile(r'^ \*\*') |
| |
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser() |
| |
| parser.add_argument('-diag', action='store', dest='diagFile', |
| help='Set the diagnostic file') |
| parser.add_argument('-warn', action='store', dest='warnFile', |
| help='Set the warning file') |
| parser.add_argument('-log', action='store', dest='logFile', |
| help='Set the log file for both diagnostics and warnings') |
| parser.add_argument('-overwrite', action='store_true', |
| help='Overwrite input filenames instead of writing different output filenames') |
| parser.add_argument('-out', action='store', dest='outDir', |
| default='out', |
| help='Set the output directory in which updated files are generated (default: out)') |
| parser.add_argument('-nowrite', action='store_true', |
| help='Do not write output files, for use with -check') |
| parser.add_argument('-check', action='store', dest='check', |
| help='Run markup checks and warn if WARN option is given, error exit if FAIL option is given') |
| parser.add_argument('-checkVUID', action='store', dest='checkVUID', |
| help='Detect duplicated VUID numbers and warn if WARN option is given, error exit if FAIL option is given') |
| parser.add_argument('-tagvu', action='store_true', |
| help='Tag un-tagged Valid Usage statements starting at the value wired into reflow.py') |
| parser.add_argument('-nextvu', action='store', dest='nextvu', type=int, |
| default=None, |
| help='Specify start VUID to use instead of the value wired into vuidCounts.py') |
| parser.add_argument('-maxvu', action='store', dest='maxvu', type=int, |
| default=None, |
| help='Specify maximum VUID instead of the value wired into vuidCounts.py') |
| parser.add_argument('-branch', action='store', dest='branch', |
| help='Specify branch to assign VUIDs for') |
| parser.add_argument('-noflow', action='store_true', dest='noflow', |
| help='Do not reflow text. Other actions may apply') |
| parser.add_argument('-margin', action='store', type=int, dest='margin', |
| default='76', |
| help='Width to reflow text, defaults to 76 characters') |
| parser.add_argument('-suffix', action='store', dest='suffix', |
| default='', |
| help='Set the suffix added to updated file names (default: none)') |
| parser.add_argument('files', metavar='filename', nargs='*', |
| help='a filename to reflow text in') |
| parser.add_argument('--version', action='version', version='%(prog)s 1.0') |
| |
| args = parser.parse_args() |
| |
| setLogFile(True, True, args.logFile) |
| setLogFile(True, False, args.diagFile) |
| setLogFile(False, True, args.warnFile) |
| |
| print('args.margin = ', args.margin) |
| |
| if args.overwrite: |
| logWarn("reflow.py: will overwrite all input files") |
| |
| errors = '' |
| if args.branch is None: |
| (args.branch, errors) = getBranch() |
| if args.branch is None: |
| # This is not fatal unless VUID assignment is required |
| if args.tagvu: |
| logErr('Cannot determine current git branch, so cannot assign VUIDs:', errors) |
| |
| if args.tagvu and args.nextvu is None: |
| # Moved here since vuidCounts is only needed in the internal |
| # repository |
| from vuidCounts import vuidCounts |
| |
| if args.branch not in vuidCounts: |
| logErr('Branch', args.branch, 'not in vuidCounts, cannot continue') |
| maxVUID = vuidCounts[args.branch][1] |
| startVUID = vuidCounts[args.branch][2] |
| args.nextvu = startVUID |
| args.maxvu = maxVUID |
| |
| if args.nextvu is not None: |
| logWarn('Tagging untagged Valid Usage statements starting at', args.nextvu) |
| |
| # Count of markup check warnings encountered |
| # This is added to the argparse structure |
| args.warnCount = 0 |
| |
| # Dictionary of VUID numbers found, containing a list of (file, line) on |
| # which that number was found |
| # This is added to the argparse structure |
| args.vuidDict = {} |
| |
| # If no files are specified, reflow the entire specification chapters folder |
| if not args.files: |
| folder_to_reflow = conventions.spec_reflow_path |
| logWarn('Reflowing all asciidoc files under', folder_to_reflow) |
| reflowAllAdocFiles(folder_to_reflow, args) |
| else: |
| for file in args.files: |
| reflowFile(file, args) |
| |
| if args.warnCount > 0: |
| if args.check == 'FAIL': |
| logErr('Failed with', args.warnCount, 'markup errors detected.\n' + |
| 'To fix these, you can take actions such as:\n' + |
| ' * Moving conditionals outside VU start / end without changing VU meaning\n' + |
| ' * Refactor conditional text using terminology defined conditionally outside the VU itself\n' + |
| ' * Remove the conditional (allowable when this just affects command / structure / enum names)\n') |
| else: |
| logWarn('Total warning count for markup issues is', args.warnCount) |
| |
| # Look for duplicated VUID numbers |
| if args.checkVUID: |
| dupVUIDs = 0 |
| for vuid in sorted(args.vuidDict): |
| found = args.vuidDict[vuid] |
| if len(found) > 1: |
| logWarn('Duplicate VUID number {} found in files:'.format(vuid)) |
| for (file, line) in found: |
| logWarn(' {}: {}'.format(file, line)) |
| dupVUIDs = dupVUIDs + 1 |
| |
| if dupVUIDs > 0: |
| if args.checkVUID == 'FAIL': |
| logErr('Failed with', dupVUIDs, 'duplicated VUID numbers found.\n' + |
| 'To fix this, either convert these to commonvalidity VUs if possible, or strip\n' + |
| 'the VUIDs from all but one of the duplicates and regenerate new ones.') |
| else: |
| logWarn('Total number of duplicated VUID numbers is', dupVUIDs) |
| |
| if args.nextvu is not None and args.nextvu != startVUID: |
| # Update next free VUID to assign |
| vuidCounts[args.branch][2] = args.nextvu |
| try: |
| reflow_count_file_path = os.path.dirname(os.path.realpath(__file__)) |
| reflow_count_file_path += '/vuidCounts.py' |
| reflow_count_file = open(reflow_count_file_path, 'w', encoding='utf8') |
| print('# Do not edit this file!', file=reflow_count_file) |
| print('# VUID ranges reserved for branches', file=reflow_count_file) |
| print('# Key is branch name, value is [ start, end, nextfree ]', file=reflow_count_file) |
| print('vuidCounts = {', file=reflow_count_file) |
| for key in sorted(vuidCounts): |
| print(" '{}': [ {}, {}, {} ],".format( |
| key, |
| vuidCounts[key][0], |
| vuidCounts[key][1], |
| vuidCounts[key][2]), |
| file=reflow_count_file) |
| print('}', file=reflow_count_file) |
| reflow_count_file.close() |
| except: |
| logWarn('Cannot open output count file vuidCounts.py', ':', sys.exc_info()[0]) |
