SR-3052. Incorrect string result for String methods with non latin symbols (#706)
* added SR-3052 tests
* expanded non latin string test
* [SR-3052] utf8 encoding fix
diff --git a/Foundation/NSCFString.swift b/Foundation/NSCFString.swift
index e12e0fa..4558549 100644
--- a/Foundation/NSCFString.swift
+++ b/Foundation/NSCFString.swift
@@ -140,19 +140,18 @@
}
internal func _CFSwiftStringGetBytes(_ str: AnyObject, encoding: CFStringEncoding, range: CFRange, buffer: UnsafeMutablePointer<UInt8>?, maxBufLen: CFIndex, usedBufLen: UnsafeMutablePointer<CFIndex>?) -> CFIndex {
+ let convertedLength: CFIndex
switch encoding {
// TODO: Don't treat many encodings like they are UTF8
case CFStringEncoding(kCFStringEncodingUTF8), CFStringEncoding(kCFStringEncodingISOLatin1), CFStringEncoding(kCFStringEncodingMacRoman), CFStringEncoding(kCFStringEncodingASCII), CFStringEncoding(kCFStringEncodingNonLossyASCII):
- let encodingView = (str as! NSString)._swiftObject.utf8
- let start = encodingView.startIndex
+ let encodingView = (str as! NSString).substring(with: NSRange(range)).utf8
if let buffer = buffer {
- for idx in 0..<range.length {
- let characterIndex = encodingView.index(start, offsetBy: idx + range.location)
- let character = encodingView[characterIndex]
+ for (idx, character) in encodingView.enumerated() {
buffer.advanced(by: idx).initialize(to: character)
}
}
- usedBufLen?.pointee = range.length
+ usedBufLen?.pointee = encodingView.count
+ convertedLength = encodingView.count
case CFStringEncoding(kCFStringEncodingUTF16):
let encodingView = (str as! NSString)._swiftObject.utf16
@@ -169,13 +168,13 @@
}
// Every character was 2 bytes
usedBufLen?.pointee = range.length * 2
-
+ convertedLength = range.length
default:
fatalError("Attempted to get bytes of a Swift string using an unsupported encoding")
}
- return range.length
+ return convertedLength
}
internal func _CFSwiftStringCreateWithSubstring(_ str: AnyObject, range: CFRange) -> Unmanaged<AnyObject> {
diff --git a/TestFoundation/TestNSString.swift b/TestFoundation/TestNSString.swift
index 50925e5..ebb8fab 100644
--- a/TestFoundation/TestNSString.swift
+++ b/TestFoundation/TestNSString.swift
@@ -81,7 +81,11 @@
("test_resolvingSymlinksInPath", test_resolvingSymlinksInPath),
("test_expandingTildeInPath", test_expandingTildeInPath),
("test_standardizingPath", test_standardizingPath),
- ("test_removingPercentEncoding", test_removingPercentEncoding),
+ ("test_addingPercentEncoding", test_addingPercentEncoding),
+ ("test_removingPercentEncodingInLatin", test_removingPercentEncodingInLatin),
+ ("test_removingPercentEncodingInNonLatin", test_removingPercentEncodingInNonLatin),
+ ("test_removingPersentEncodingWithoutEncoding", test_removingPersentEncodingWithoutEncoding),
+ ("test_addingPercentEncodingAndBack", test_addingPercentEncodingAndBack),
("test_stringByAppendingPathExtension", test_stringByAppendingPathExtension),
("test_deletingPathExtension", test_deletingPathExtension),
("test_ExternalRepresentation", test_ExternalRepresentation),
@@ -867,13 +871,66 @@
}
}
- func test_removingPercentEncoding() {
+ func test_addingPercentEncoding() {
+ let s1 = "a b".addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+ XCTAssertEqual(s1, "a%20b")
+
+ let s2 = "\u{0434}\u{043E}\u{043C}".addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+ XCTAssertEqual(s2, "%D0%B4%D0%BE%D0%BC")
+ }
+
+ func test_removingPercentEncodingInLatin() {
let s1 = "a%20b".removingPercentEncoding
XCTAssertEqual(s1, "a b")
let s2 = "a%1 b".removingPercentEncoding
XCTAssertNil(s2, "returns nil for a string with an invalid percent encoding")
}
+ func test_removingPercentEncodingInNonLatin() {
+ let s1 = "\u{043C}\u{043E}\u{0439}%20\u{0434}\u{043E}\u{043C}".removingPercentEncoding
+ XCTAssertEqual(s1, "\u{043C}\u{043E}\u{0439} \u{0434}\u{043E}\u{043C}")
+
+ let s2 = "%D0%B4%D0%BE%D0%BC".removingPercentEncoding
+ XCTAssertEqual(s2, "\u{0434}\u{043E}\u{043C}")
+
+ let s3 = "\u{00E0}a%1 b".removingPercentEncoding
+ XCTAssertNil(s3, "returns nil for a string with an invalid percent encoding")
+ }
+
+ func test_removingPersentEncodingWithoutEncoding() {
+ let cyrillicString = "\u{0434}\u{043E}\u{043C}"
+ let cyrillicEscapedString = cyrillicString.removingPercentEncoding
+ XCTAssertEqual(cyrillicString, cyrillicEscapedString)
+
+ let chineseString = "\u{623F}\u{5B50}"
+ let chineseEscapedString = chineseString.removingPercentEncoding
+ XCTAssertEqual(chineseString, chineseEscapedString)
+
+ let arabicString = "\u{0645}\u{0646}\u{0632}\u{0644}"
+ let arabicEscapedString = arabicString.removingPercentEncoding
+ XCTAssertEqual(arabicString, arabicEscapedString)
+
+ let randomString = "\u{00E0}\u{00E6}"
+ let randomEscapedString = randomString.removingPercentEncoding
+ XCTAssertEqual(randomString, randomEscapedString)
+
+ let latinString = "home"
+ let latinEscapedString = latinString.removingPercentEncoding
+ XCTAssertEqual(latinString, latinEscapedString)
+ }
+
+ func test_addingPercentEncodingAndBack() {
+ let latingString = "a b"
+ let escapedLatingString = latingString.addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+ let returnedLatingString = escapedLatingString?.removingPercentEncoding
+ XCTAssertEqual(returnedLatingString, latingString)
+
+ let cyrillicString = "\u{0434}\u{043E}\u{043C}"
+ let escapedCyrillicString = cyrillicString.addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+ let returnedCyrillicString = escapedCyrillicString?.removingPercentEncoding
+ XCTAssertEqual(returnedCyrillicString, cyrillicString)
+ }
+
func test_stringByAppendingPathExtension() {
let values = [
NSString(string: "/tmp/scratch.old") : "/tmp/scratch.old.tiff",
