Merge branch 'main' into HEAD
diff --git a/Lib/fontTools/ttLib/tables/distillery/__init__.py b/Lib/fontTools/ttLib/tables/distillery/__init__.py
new file mode 100644
index 0000000..d15706d
--- /dev/null
+++ b/Lib/fontTools/ttLib/tables/distillery/__init__.py
@@ -0,0 +1,141 @@
+"""This module helps with distilling a set of objects (aka tables) linked together through
+a set of object references (represented eventually by offsets) into a bytestream in a compact
+and efficient manner. This is useful for compiling tables like GSUB/GPOS/GDEF and other
+tables represented in the otData.py module."""
+
+from __future__ import print_function, division, absolute_import
+from fontTools.misc.py23 import *
+
+from fibonacci_heap_mod import Fibonacci_heap
+
+
+def sortMeOut(table):
+ distillery = Distillery()
+ return distillery.distill(table)
+
+class ObjectSet(object):
+
+ def __init__(self):
+ self._s = set()
+ self._d = dict()
+
+ def add(self, obj):
+ self._s.add(id(obj))
+ self._d[id(obj)] = obj
+
+ def remove(self, obj):
+ self._s.remove(id(obj))
+ del self._d[id(obj)]
+
+ def issubset(self, other):
+ assert type(other) == ObjectSet
+ return self._s.issubset(other._s)
+
+ def __iter__(self):
+ return iter(self._d.values())
+
+ def __contains__(self, obj):
+ return id(obj) in self._s
+
+ def __bool__(self):
+ return bool(self._s)
+
+ __nonzero__ = __bool__
+
+ def __len__(self):
+ return len(self._s)
+
+ def __repr__(self):
+ return repr(self._s)
+
+class Distillery(object):
+
+ def __init__(self):
+ self.packedS = ObjectSet()
+ self.readyQ = Fibonacci_heap()
+ self.readyS = ObjectSet()
+ #self.awaitQ = Fibonacci_heap()
+ #self.awaitS = ObjectSet()
+
+ def _itemPriority(self, item):
+ leashLen = (65536)**2 if item.offsetSize == 4 else 65536
+ packedParentsPos = [p.pos for p in item.parents if p in self.packedS]
+ leashStart = min(packedParentsPos) if packedParentsPos else (65536)**2 * 2
+ itemLen = len(item)
+ return (leashStart + leashLen + itemLen)# / 4
+
+ def _itemIsReady(self, item):
+ return item.parents.issubset(self.packedS)
+
+ def _readyEnqueue(self, item):
+ item.node = self.readyQ.enqueue(item, self._itemPriority(item))
+ self.readyS.add(item)
+
+ #def _awaitEnqueue(self, item):
+ # item.node = self.awaitQ.enqueue(item, self._itemPriority(item))
+ # self.awaitS.add(item)
+
+ #def _awaitDelete(self, item):
+ # assert item in self.awaitS
+ # self.awaitS.remove(item)
+ # self.awaitQ.delete(item.node)
+ # del item.node
+
+ def _setParents(self, root, done):
+ done.add(root)
+ root.parents = ObjectSet()
+ for item in root.items:
+ if not hasattr(item, 'getData'):
+ continue
+ if item not in done:
+ self._setParents(item, done)
+ item.parents.add(root)
+
+ def distill(self, root):
+ self._setParents(root, ObjectSet())
+
+ out = []
+
+ self._readyEnqueue(root)
+
+ pos = 0
+ while self.readyQ:
+ entry = self.readyQ.dequeue_min()
+ obj = entry.get_value()
+ self.readyS.remove(obj)
+ prio = entry.get_priority()
+ del entry
+ del obj.node
+
+ out.append(obj)
+ obj.pos = pos
+ self.packedS.add(obj)
+
+ for item in obj.items:
+ if not hasattr(item, "getData"):
+ continue
+
+ assert item not in self.packedS
+ if item in self.readyS:
+ continue
+
+ if self._itemIsReady(item):
+ #if hasattr(item, 'node'):
+ # self._awaitDelete(item)
+
+ self._readyEnqueue(item)
+ else:
+ pass
+ #if hasattr(item, 'node'):
+ # self.awaitQ.decrease_key(item.node, self._itemPriority(item))
+ #else:
+ # self._awaitEnqueue(item)
+
+ pos += len(obj)
+
+ #assert not self.awaitS, self.awaitS
+ #assert not self.awaitQ, self.awaitQ
+ assert not self.readyS, self.readyS
+ assert not self.readyQ, self.readyQ
+
+ return out
diff --git a/Lib/fontTools/ttLib/tables/distillery/dot.py b/Lib/fontTools/ttLib/tables/distillery/dot.py
new file mode 100644
index 0000000..f89dbf3
--- /dev/null
+++ b/Lib/fontTools/ttLib/tables/distillery/dot.py
@@ -0,0 +1,93 @@
+"""Generates a Graphviz DOT file representing the (directed) graph of OTTableWriter objects."""
+
+from __future__ import print_function, division, absolute_import
+from fontTools.misc.py23 import *
+from fontTools.ttLib import TTFont
+from fontTools.ttLib.tables.otBase import OTTableWriter
+import logging
+import sys
+import math
+from collections import deque
+
+log = logging.getLogger(__name__)
+
+def writerId(writer):
+ s = '"' + writer.name
+
+ if hasattr(writer, 'repeatIndex'):
+ s += ' ' + str(writer.repeatIndex)
+
+ s += '\n' + str(id(writer))
+ return s + '"'
+
+def writerLabel(writer):
+ size = writer.getDataLength()
+ return "%s\n%s bytes" % (writer.name, size)
+
+def graph(font, tableTags, stream=sys.stdout):
+ f = stream
+ for tableTag in tableTags:
+ log.debug("Processing %s", tableTag)
+ table = font[tableTag].table
+
+ writer = OTTableWriter(tableTag=tableTag)
+ writer.name = tableTag
+ table.compile(writer, font)
+
+ f.write("digraph %s {\n" % tableTag)
+ f.write(" rankdir=LR;\n")
+
+ names = {}
+ queue = deque()
+
+ queue.append (writer)
+ names[id(writer)] = tableTag
+ f.write('graph [root=%s];\n' % tableTag)
+ f.write('node [margin=0,width=0,height=0];\n')
+ while queue:
+ writer = queue.popleft()
+ lhs = names[id(writer)]
+ label = writerLabel(writer)
+ size = writer.getDataLength()
+ fontsize = 5+math.log(size) * 4
+ color = "green" if len(writer.parents) > 1 else "black"
+ f.write('%s [fontsize=%s,label="%s",color=%s]\n' % (lhs, fontsize, label, color))
+
+ for item in writer.items:
+ if not hasattr(item, "getData"):
+ continue
+ offsetSize = item.offsetSize
+ rhs = writerId(item)
+ f.write(' '+lhs+' -> '+rhs+';\n')
+
+ if id(item) not in names:
+ names[id(item)] = rhs
+ queue.append(item)
+
+ f.write("}\n")
+
+
+
+def main(args=None):
+ from fontTools import configLogger
+
+ if args is None:
+ args = sys.argv[1:]
+
+ # configure the library logger (for >= WARNING)
+ configLogger()
+ # comment this out to enable debug messages from logger
+ # log.setLevel(logging.DEBUG)
+
+ if len(args) < 1:
+ print("usage: fonttools ttLib.tables.distillery.dot font-file.ttf [table-tags...]", file=sys.stderr)
+ sys.exit(1)
+
+ font = TTFont(args[0])
+ tags = args[1:]
+
+ graph(font, tags)
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main())
diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py
index 0e74812..fe77196 100644
--- a/Lib/fontTools/ttLib/tables/otBase.py
+++ b/Lib/fontTools/ttLib/tables/otBase.py
@@ -4,6 +4,7 @@
import array
import struct
import logging
+from . import distillery
log = logging.getLogger(__name__)
@@ -66,6 +67,7 @@
while True:
try:
writer = OTTableWriter(tableTag=self.tableTag)
+ writer.name = self.tableTag
self.table.compile(writer, font)
return writer.getAllData()
@@ -222,24 +224,15 @@
"""Helper class to gather and assemble data for OpenType tables."""
- def __init__(self, localState=None, tableTag=None, offsetSize=2):
- self.items = []
- self.pos = None
+ def __init__(self, localState=None, tableTag=None, name='', offsetSize=2):
self.localState = localState
self.tableTag = tableTag
+ self.name = name
+ self.items = []
+ self.pos = None
self.offsetSize = offsetSize
self.parent = None
- # DEPRECATED: 'longOffset' is kept as a property for backward compat with old code.
- # You should use 'offsetSize' instead (2, 3 or 4 bytes).
- @property
- def longOffset(self):
- return self.offsetSize == 4
-
- @longOffset.setter
- def longOffset(self, value):
- self.offsetSize = 4 if value else 2
-
def __setitem__(self, name, value):
state = self.localState.copy() if self.localState else dict()
state[name] = value
@@ -265,13 +258,16 @@
l = l + len(item)
return l
+ def __len__(self):
+ if not hasattr(self, '_length'):
+ self._length = self.getDataLength()
+ return self._length
+
def getData(self):
"""Assemble the data for this writer/table, without subtables."""
items = list(self.items) # make a shallow copy
pos = self.pos
- numItems = len(items)
- for i in range(numItems):
- item = items[i]
+ for i,item in enumerate(items):
if hasattr(item, "getData"):
if item.offsetSize == 4:
@@ -292,8 +288,10 @@
return bytesjoin(items)
def __hash__(self):
- # only works after self._doneWriting() has been called
- return hash(self.items)
+ if not hasattr(self, '_hash'):
+ items = tuple(id(x) if hasattr(x, 'getData') else x for x in self.items)
+ self._hash = hash(items)
+ return self._hash
def __ne__(self, other):
result = self.__eq__(other)
@@ -302,36 +300,47 @@
def __eq__(self, other):
if type(self) != type(other):
return NotImplemented
- return self.offsetSize == other.offsetSize and self.items == other.items
+ if len(self.items) != len(other.items):
+ return False
+ if self.offsetSize != other.offsetSize:
+ return False
+ return all(x is y if hasattr(x, 'getData') else x == y for x,y in zip(self.items, other.items))
+
+ def _writeCountData(self):
+ # Convert CountData references to data string items.
+ items = self.items
+ for i in range(len(items)):
+ item = items[i]
+ if hasattr(item, "getCountData"):
+ items[i] = item.getCountData()
+ elif hasattr(item, "getData"):
+ item._writeCountData()
def _doneWriting(self, internedTables):
- # Convert CountData references to data string items
- # collapse duplicate table references to a unique entry
+ # Collapse duplicate table references to a unique entry
# "tables" are OTTableWriter objects.
- # For Extension Lookup types, we can
- # eliminate duplicates only within the tree under the Extension Lookup,
- # as offsets may exceed 64K even between Extension LookupTable subtables.
- isExtension = hasattr(self, "Extension")
-
# Certain versions of Uniscribe reject the font if the GSUB/GPOS top-level
# arrays (ScriptList, FeatureList, LookupList) point to the same, possibly
# empty, array. So, we don't share those.
# See: https://github.com/fonttools/fonttools/issues/518
dontShare = hasattr(self, 'DontShare')
- if isExtension:
- internedTables = {}
-
items = self.items
for i in range(len(items)):
item = items[i]
if hasattr(item, "getCountData"):
items[i] = item.getCountData()
- elif hasattr(item, "getData"):
- item._doneWriting(internedTables)
- if not dontShare:
- items[i] = item = internedTables.setdefault(item, item)
+ if not hasattr(item, "getData"):
+ continue
+
+ item._doneWriting(internedTables)
+ if not dontShare:
+ internedItem = internedTables.get(item)
+ if internedItem:
+ item = items[i] = internedItem
+ else:
+ internedTables[item] = item
self.items = tuple(items)
def _gatherTables(self, tables, extTables, done):
@@ -387,44 +396,21 @@
# Item is already written out by other parent
pass
- selfTables.append(self)
+ self.items = tuple(items)
def getAllData(self):
"""Assemble all data, including all subtables."""
+ self._writeCountData()
internedTables = {}
self._doneWriting(internedTables)
- tables = []
- extTables = []
- done = {}
- self._gatherTables(tables, extTables, done)
- tables.reverse()
- extTables.reverse()
- # Gather all data in two passes: the absolute positions of all
- # subtable are needed before the actual data can be assembled.
- pos = 0
- for table in tables:
- table.pos = pos
- pos = pos + table.getDataLength()
-
- for table in extTables:
- table.pos = pos
- pos = pos + table.getDataLength()
-
- data = []
- for table in tables:
- tableData = table.getData()
- data.append(tableData)
-
- for table in extTables:
- tableData = table.getData()
- data.append(tableData)
-
- return bytesjoin(data)
+ tables = distillery.sortMeOut(self)
+ return bytesjoin(table.getData() for table in tables)
# interface for gathering data, as used by table.compile()
- def getSubWriter(self, offsetSize=2):
- subwriter = self.__class__(self.localState, self.tableTag, offsetSize=offsetSize)
+ def getSubWriter(self, offsetSize=2, name=''):
+ subwriter = self.__class__(self.localState, self.tableTag, name, offsetSize=offsetSize)
+ subwriter.offsetSize = offsetSize
subwriter.parent = self # because some subtables have idential values, we discard
# the duplicates under the getAllData method. Hence some
# subtable writers can have more than one parent writer.
@@ -512,23 +498,23 @@
if hasattr(item, 'repeatIndex'):
itemIndex = item.repeatIndex
if self.name == 'SubTable':
- LookupListIndex = self.parent.repeatIndex
+ LookupListIndex = self.parents[0].repeatIndex
SubTableIndex = self.repeatIndex
elif self.name == 'ExtSubTable':
- LookupListIndex = self.parent.parent.repeatIndex
- SubTableIndex = self.parent.repeatIndex
+ LookupListIndex = self.parents[0].parents[0].repeatIndex
+ SubTableIndex = self.parents[0].repeatIndex
else: # who knows how far below the SubTable level we are! Climb back up to the nearest subtable.
itemName = ".".join([self.name, itemName])
- p1 = self.parent
+ p1 = self.parents[0]
while p1 and p1.name not in ['ExtSubTable', 'SubTable']:
itemName = ".".join([p1.name, itemName])
- p1 = p1.parent
+ p1 = p1.parents[0]
if p1:
if p1.name == 'ExtSubTable':
- LookupListIndex = p1.parent.parent.repeatIndex
- SubTableIndex = p1.parent.repeatIndex
+ LookupListIndex = p1.parents[0].parents[0].repeatIndex
+ SubTableIndex = p1.parents[0].repeatIndex
else:
- LookupListIndex = p1.parent.repeatIndex
+ LookupListIndex = p1.parents[0].repeatIndex
SubTableIndex = p1.repeatIndex
return OverflowErrorRecord( (self.tableTag, LookupListIndex, SubTableIndex, itemName, itemIndex) )
@@ -966,7 +952,7 @@
value = getattr(valueRecord, name, 0)
if isDevice:
if value:
- subWriter = writer.getSubWriter()
+ subWriter = writer.getSubWriter(name=name)
writer.writeSubTable(subWriter)
value.compile(subWriter, font)
else:
diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py
index 058813a..9cd9bfc 100644
--- a/Lib/fontTools/ttLib/tables/otConverters.py
+++ b/Lib/fontTools/ttLib/tables/otConverters.py
@@ -637,8 +637,7 @@
if value is None:
self.writeNullOffset(writer)
else:
- subWriter = writer.getSubWriter(offsetSize=self.staticSize)
- subWriter.name = self.name
+ subWriter = writer.getSubWriter(offsetSize=self.staticSize, name = self.name)
if repeatIndex is not None:
subWriter.repeatIndex = repeatIndex
writer.writeSubTable(subWriter)
@@ -1006,6 +1005,7 @@
lookup.write(lookupWriter, font, tableDict, offsetByGlyph, None)
dataWriter = writer.getSubWriter(offsetSize=4)
+
writer.writeSubTable(lookupWriter)
writer.writeSubTable(dataWriter)
for d in compiledData:
diff --git a/Lib/fontTools/ttLib/tables/otTables.py b/Lib/fontTools/ttLib/tables/otTables.py
index 679df75..c2d53d1 100644
--- a/Lib/fontTools/ttLib/tables/otTables.py
+++ b/Lib/fontTools/ttLib/tables/otTables.py
@@ -1082,12 +1082,6 @@
alts = AlternateSet()
alts.Alternate = set
alternates.append(alts)
- # a special case to deal with the fact that several hundred Adobe Japan1-5
- # CJK fonts will overflow an offset if the coverage table isn't pushed to the end.
- # Also useful in that when splitting a sub-table because of an offset overflow
- # I don't need to calculate the change in the subtable offset due to the change in the coverage table size.
- # Allows packing more rules in subtable.
- self.sortCoverageLast = 1
return {"Coverage": cov, "AlternateSet": alternates}
def toXML2(self, xmlWriter, font):
@@ -1171,10 +1165,6 @@
for lig in set:
ligs.append(lig)
ligSets.append(ligSet)
- # Useful in that when splitting a sub-table because of an offset overflow
- # I don't need to calculate the change in subtabl offset due to the coverage table size.
- # Allows packing more rules in subtable.
- self.sortCoverageLast = 1
return {"Coverage": cov, "LigatureSet": ligSets}
def toXML2(self, xmlWriter, font):
@@ -1709,6 +1699,7 @@
def splitAlternateSubst(oldSubTable, newSubTable, overflowRecord):
ok = 1
+ newSubTable.Format = oldSubTable.Format
if hasattr(oldSubTable, 'sortCoverageLast'):
newSubTable.sortCoverageLast = oldSubTable.sortCoverageLast
