Source/JavaScriptCore/generateYarrCanonicalizeUnicode

#! /usr/bin/python

# Copyright (C) 2016 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer. 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# This tool processes the Unicode Character Database file CaseFolding.txt to create
# canonicalization table as decribed in ECMAScript 6 standard in section
# "21.2.2.8.2 Runtime Semantics: Canonicalize()", step 2.

import optparse
import re
import sys
from sets import Set

header = """/*
* Copyright (C) 2016 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1.  Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer. 
* 2.  Redistributions in binary form must reproduce the above copyright
*     notice, this list of conditions and the following disclaimer in the
*     documentation and/or other materials provided with the distribution. 
*
* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// DO NO EDIT! - This file was generated by generateYarrCanonicalizeUnicode

#include "config.h"
#include "YarrCanonicalize.h"

namespace JSC { namespace Yarr {

"""

footer = """} } // JSC::Yarr
"""

MaxUnicode = 0x10ffff
commonAndSimpleLinesRE = re.compile(r"(?P<code>[0-9A-F]+)\s*;\s*[CS]\s*;\s*(?P<mapping>[0-9A-F]+)", re.IGNORECASE)

def openOrExit(path, mode):
    try:
        return open(path, mode)
    except IOError as e:
        print "I/O error opening {0}, ({1}): {2}".format(path, e.errno, e.strerror)
        exit(1)

class Canonicalize:
    def __init__(self):
        self.canonicalGroups = {};

    def addMapping(self, code, mapping):
        if mapping not in self.canonicalGroups:
            self.canonicalGroups[mapping] = []
        self.canonicalGroups[mapping].append(code)
        
    def readCaseFolding(self, file):
        codesSeen = Set()
        for line in file:
            line = line.split('#', 1)[0]
            line = line.rstrip()
            if (not len(line)):
                continue

            fields = commonAndSimpleLinesRE.match(line)
            if (not fields):
                continue

            code = int(fields.group('code'), 16)
            mapping = int(fields.group('mapping'), 16)

            codesSeen.add(code)
            self.addMapping(code, mapping)

        for i in range(MaxUnicode + 1):
            if i in codesSeen:
                continue;

            self.addMapping(i, i)

    def createTables(self, file):
        typeInfo = [""] * (MaxUnicode + 1)
        characterSets = []

        for mapping in sorted(self.canonicalGroups.keys()):
            characters = self.canonicalGroups[mapping]
            if len(characters) == 1:
                typeInfo[characters[0]] = "CanonicalizeUnique:0"
            else:
                characters.sort()
                if len(characters) > 2:
                    for ch in characters:
                        typeInfo[ch] = "CanonicalizeSet:%d" % len(characterSets)
                    characterSets.append(characters)
                else:
                    low = characters[0]
                    high = characters[1]
                    delta = high - low
                    if delta == 1:
                        type = "CanonicalizeAlternatingUnaligned:0" if low & 1 else "CanonicalizeAlternatingAligned:0"
                        typeInfo[low] = type
                        typeInfo[high] = type
                    else:
                        typeInfo[low] = "CanonicalizeRangeLo:%d" % delta
                        typeInfo[high] = "CanonicalizeRangeHi:%d" % delta

        rangeInfo = []
        end = 0
        while end <= MaxUnicode:
            begin = end
            type = typeInfo[end]
            while end < MaxUnicode and typeInfo[end + 1] == type:
                end = end + 1
            rangeInfo.append({"begin": begin, "end": end, "type": type})
            end = end + 1
        
        for i in range(len(characterSets)):
            characters = ""
            set = characterSets[i]
            for ch in set:
                characters = characters + "0x{character:04x}, ".format(character=ch)
            file.write("const UChar32 unicodeCharacterSet{index:d}[] = {{ {characters}0 }};\n".format(index=i, characters=characters))

        file.write("\n")
        file.write("static const size_t UNICODE_CANONICALIZATION_SETS = {setCount:d};\n".format(setCount=len(characterSets)))
        file.write("const UChar32* const unicodeCharacterSetInfo[UNICODE_CANONICALIZATION_SETS] = {\n")

        for i in range(len(characterSets)):
            file.write("    unicodeCharacterSet{setNumber:d},\n".format(setNumber=i))

        file.write("};\n")
        file.write("\n")
        file.write("const size_t UNICODE_CANONICALIZATION_RANGES = {rangeCount:d};\n".format(rangeCount=len(rangeInfo)))
        file.write("const CanonicalizationRange unicodeRangeInfo[UNICODE_CANONICALIZATION_RANGES] = {\n")

        for info in rangeInfo:
            typeAndValue = info["type"].split(":")
            file.write("    {{ 0x{begin:04x}, 0x{end:04x}, 0x{value:04x}, {type} }},\n".format(begin=info["begin"], end=info["end"], value=int(typeAndValue[1]), type=typeAndValue[0]))

        file.write("};\n")
        file.write("\n")

        
if __name__ == "__main__":
    parser = optparse.OptionParser(usage = "usage: %prog  <CaseFolding.txt> <YarrCanonicalizeUnicode.h>")
    (options, args) = parser.parse_args()

    if len(args) != 2:
        parser.error("<CaseFolding.txt> <YarrCanonicalizeUnicode.h>")

    caseFoldingTxtPath = args[0]
    canonicalizeHPath = args[1]
    caseFoldingTxtFile = openOrExit(caseFoldingTxtPath, "r")
    canonicalizeHFile = openOrExit(canonicalizeHPath, "wb")

    canonicalize = Canonicalize()
    canonicalize.readCaseFolding(caseFoldingTxtFile)

    canonicalizeHFile.write(header);
    canonicalize.createTables(canonicalizeHFile)
    canonicalizeHFile.write(footer);

    caseFoldingTxtFile.close()
    canonicalizeHFile.close()

    exit(0)