lib/root_ssl_certificates/third_party/convert_mozilla_certdata/convert_mozilla_certdata.go

// Copyright 2012 Google Inc. All Rights Reserved.
// Author: agl@chromium.org (Adam Langley)
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This utility parses Mozilla's certdata.txt and extracts a list of trusted
// certificates in PEM form.
//
// A current version of certdata.txt can be downloaded from:
//   https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt
package main
import (
  "bufio"
  "bytes"
  "crypto"
  _ "crypto/md5"
  "crypto/sha1"
  _ "crypto/sha256"
  "crypto/x509"
  "crypto/x509/pkix"
  "encoding/pem"
  "flag"
  "fmt"
  "io"
  "log"
  "os"
  "strconv"
  "strings"
  "unicode/utf8"
)
// Object represents a collection of attributes from the certdata.txt file
// which are usually either certificates or trust records.
type Object struct {
  attrs        map[string]Attribute
  startingLine int // the line number that the object started on.
}
type Attribute struct {
  attrType string
  value    []byte
}
var (
  // ignoreList maps from CKA_LABEL values (from the upstream roots file)
  // to an optional comment which is displayed when skipping matching
  // certificates.
  ignoreList map[string]string
  includedUntrustedFlag = flag.Bool("include-untrusted", false, "If set, untrusted certificates will also be included in the output")
  toFiles               = flag.Bool("to-files", false, "If set, individual certificate files will be created in the current directory")
  ignoreListFilename    = flag.String("ignore-list", "", "File containing a list of certificates to ignore")
)
func main() {
  flag.Parse()
  inFilename := "certdata.txt"
  if len(flag.Args()) == 1 {
    inFilename = flag.Arg(0)
  } else if len(flag.Args()) > 1 {
    fmt.Printf("Usage: %s [<certdata.txt file>]\n", os.Args[0])
    os.Exit(1)
  }
  ignoreList = make(map[string]string)
  if *ignoreListFilename != "" {
    ignoreListFile, err := os.Open(*ignoreListFilename)
    if err != nil {
      log.Fatalf("Failed to open ignore-list file: %s", err)
    }
    parseIgnoreList(ignoreListFile)
    ignoreListFile.Close()
  }
  inFile, err := os.Open(inFilename)
  if err != nil {
    log.Fatalf("Failed to open input file: %s", err)
  }
  license, cvsId, objects := parseInput(inFile)
  inFile.Close()
  if !*toFiles {
    os.Stdout.WriteString(license)
    if len(cvsId) > 0 {
      os.Stdout.WriteString("CVS_ID " + cvsId + "\n")
    }
  }
  outputTrustedCerts(os.Stdout, objects)
}
// parseIgnoreList parses the ignore-list file into ignoreList
func parseIgnoreList(ignoreListFile io.Reader) {
  in := bufio.NewReader(ignoreListFile)
  var lineNo int
  for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) {
    if split := strings.SplitN(line, "#", 2); len(split) == 2 {
      // this line has an additional comment
      ignoreList[strings.TrimSpace(split[0])] = strings.TrimSpace(split[1])
    } else {
      ignoreList[line] = ""
    }
  }
}
// parseInput parses a certdata.txt file into it's license blob, the CVS id (if
// included) and a set of Objects.
func parseInput(inFile io.Reader) (license, cvsId string, objects []*Object) {
  in := bufio.NewReader(inFile)
  var lineNo int
  // Discard anything prior to the license block.
  for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) {
    if strings.Contains(line, "This Source Code") {
      license += line
      license += "\n"
      break
    }
  }
  if len(license) == 0 {
    log.Fatalf("Read whole input and failed to find beginning of license")
  }
  // Now collect the license block.
  // certdata.txt from hg.mozilla.org no longer contains CVS_ID.
  for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) {
    if strings.Contains(line, "CVS_ID") || len(line) == 0 {
      break
    }
    license += line
    license += "\n"
  }
  var currentObject *Object
  var beginData bool
  for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) {
    if len(line) == 0 || line[0] == '#' {
      continue
    }
    if strings.HasPrefix(line, "CVS_ID ") {
      cvsId = line[7:]
      continue
    }
    if line == "BEGINDATA" {
      beginData = true
      continue
    }
    words := strings.Fields(line)
    var value []byte
    if len(words) == 2 && words[1] == "MULTILINE_OCTAL" {
      startingLine := lineNo
      var ok bool
      value, ok = readMultilineOctal(in, &lineNo)
      if !ok {
        log.Fatalf("Failed to read octal value starting at line %d", startingLine)
      }
    } else if len(words) < 3 {
      log.Fatalf("Expected three or more values on line %d, but found %d", lineNo, len(words))
    } else {
      value = []byte(strings.Join(words[2:], " "))
    }
    if words[0] == "CKA_CLASS" {
      // Start of a new object.
      if currentObject != nil {
        objects = append(objects, currentObject)
      }
      currentObject = new(Object)
      currentObject.attrs = make(map[string]Attribute)
      currentObject.startingLine = lineNo
    }
    if currentObject == nil {
      log.Fatalf("Found attribute on line %d which appears to be outside of an object", lineNo)
    }
    currentObject.attrs[words[0]] = Attribute{
      attrType: words[1],
      value:    value,
    }
  }
  if !beginData {
    log.Fatalf("Read whole input and failed to find BEGINDATA")
  }
  if currentObject != nil {
    objects = append(objects, currentObject)
  }
  return
}
// outputTrustedCerts writes a series of PEM encoded certificates to out by
// finding certificates and their trust records in objects.
func outputTrustedCerts(out *os.File, objects []*Object) {
  certs := filterObjectsByClass(objects, "CKO_CERTIFICATE")
  trusts := filterObjectsByClass(objects, "CKO_NSS_TRUST")
  filenames := make(map[string]bool)
  for _, cert := range certs {
    derBytes := cert.attrs["CKA_VALUE"].value
    hash := sha1.New()
    hash.Write(derBytes)
    digest := hash.Sum(nil)
    label := string(cert.attrs["CKA_LABEL"].value)
    if comment, present := ignoreList[strings.Trim(label, "\"")]; present {
      var sep string
      if len(comment) > 0 {
        sep = ": "
      }
      log.Printf("Skipping explicitly ignored certificate: %s%s%s", label, sep, comment)
      continue
    }
    x509, err := x509.ParseCertificate(derBytes)
    if err != nil {
      // This is known to occur because of a broken certificate in NSS.
      // https://bugzilla.mozilla.org/show_bug.cgi?id=707995
      log.Printf("Failed to parse certificate starting on line %d: %s", cert.startingLine, err)
      continue
    }
    // TODO(agl): wtc tells me that Mozilla might get rid of the
    // SHA1 records in the future and use issuer and serial number
    // to match trust records to certificates (which is what NSS
    // currently uses). This needs some changes to the crypto/x509
    // package to keep the raw names around.
    var trust *Object
    for _, possibleTrust := range trusts {
      if bytes.Equal(digest, possibleTrust.attrs["CKA_CERT_SHA1_HASH"].value) {
        trust = possibleTrust
        break
      }
    }
    if trust == nil {
      log.Fatalf("No trust found for certificate object starting on line %d (sha1: %x)", cert.startingLine, digest)
    }
    trustType := trust.attrs["CKA_TRUST_SERVER_AUTH"].value
    if len(trustType) == 0 {
      log.Fatalf("No CKA_TRUST_SERVER_AUTH found in trust starting at line %d", trust.startingLine)
    }
    var trusted bool
    switch string(trustType) {
    case "CKT_NSS_NOT_TRUSTED":
      // An explicitly distrusted cert
      trusted = false
    case "CKT_NSS_TRUSTED_DELEGATOR":
      // A cert trusted for issuing SSL server certs.
      trusted = true
    case "CKT_NSS_TRUST_UNKNOWN", "CKT_NSS_MUST_VERIFY_TRUST":
      // A cert not trusted for issuing SSL server certs, but is trusted for other purposes.
      trusted = false
    default:
      log.Fatalf("Unknown trust value '%s' found for trust record starting on line %d", trustType, trust.startingLine)
    }
    if !trusted && !*includedUntrustedFlag {
      continue
    }
    block := &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}
    if *toFiles {
      if strings.HasPrefix(label, "\"") {
        label = label[1:]
      }
      if strings.HasSuffix(label, "\"") {
        label = label[:len(label)-1]
      }
      // The label may contain hex-escaped, UTF-8 charactors.
      label = unescapeLabel(label)
      label = strings.Replace(label, " ", "_", -1)
      label = strings.Replace(label, "/", "_", -1)
      filename := label
      for i := 2; ; i++ {
        if _, ok := filenames[filename]; !ok {
          break
        }
        filename = label + "-" + strconv.Itoa(i)
      }
      filenames[filename] = true
      file, err := os.Create(filename + ".pem")
      if err != nil {
        log.Fatalf("Failed to create output file: %s\n", err)
      }
      pem.Encode(file, block)
      file.Close()
      out.WriteString(filename + ".pem\n")
      continue
    }
    out.WriteString("\n")
    if !trusted {
      out.WriteString("# NOT TRUSTED FOR SSL\n")
    }
    out.WriteString("# Issuer: " + nameToString(x509.Issuer) + "\n")
    out.WriteString("# Subject: " + nameToString(x509.Subject) + "\n")
    out.WriteString("# Label: " + label + "\n")
    out.WriteString("# Serial: " + x509.SerialNumber.String() + "\n")
    out.WriteString("# MD5 Fingerprint: " + fingerprintString(crypto.MD5, x509.Raw) + "\n")
    out.WriteString("# SHA1 Fingerprint: " + fingerprintString(crypto.SHA1, x509.Raw) + "\n")
    out.WriteString("# SHA256 Fingerprint: " + fingerprintString(crypto.SHA256, x509.Raw) + "\n")
    pem.Encode(out, block)
  }
}
// nameToString converts name into a string representation containing the
// CommonName, Organization and OrganizationalUnit.
func nameToString(name pkix.Name) string {
  ret := ""
  if len(name.CommonName) > 0 {
    ret += "CN=" + name.CommonName
  }
  if org := strings.Join(name.Organization, "/"); len(org) > 0 {
    if len(ret) > 0 {
      ret += " "
    }
    ret += "O=" + org
  }
  if orgUnit := strings.Join(name.OrganizationalUnit, "/"); len(orgUnit) > 0 {
    if len(ret) > 0 {
      ret += " "
    }
    ret += "OU=" + orgUnit
  }
  return ret
}
// filterObjectsByClass returns a subset of in where each element has the given
// class.
func filterObjectsByClass(in []*Object, class string) (out []*Object) {
  for _, object := range in {
    if string(object.attrs["CKA_CLASS"].value) == class {
      out = append(out, object)
    }
  }
  return
}
// readMultilineOctal converts a series of lines of octal values into a slice
// of bytes.
func readMultilineOctal(in *bufio.Reader, lineNo *int) ([]byte, bool) {
  var value []byte
  for line, eof := getLine(in, lineNo); !eof; line, eof = getLine(in, lineNo) {
    if line == "END" {
      return value, true
    }
    for _, octalStr := range strings.Split(line, "\\") {
      if len(octalStr) == 0 {
        continue
      }
      v, err := strconv.ParseUint(octalStr, 8, 8)
      if err != nil {
        log.Printf("error converting octal string '%s' on line %d", octalStr, *lineNo)
        return nil, false
      }
      value = append(value, byte(v))
    }
  }
  // Missing "END"
  return nil, false
}
// getLine reads the next line from in, aborting in the event of an error.
func getLine(in *bufio.Reader, lineNo *int) (string, bool) {
  *lineNo++
  line, isPrefix, err := in.ReadLine()
  if err == io.EOF {
    return "", true
  }
  if err != nil {
    log.Fatalf("I/O error while reading input: %s", err)
  }
  if isPrefix {
    log.Fatalf("Line too long while reading line %d", *lineNo)
  }
  return string(line), false
}
func fingerprintString(hashFunc crypto.Hash, data []byte) string {
  hash := hashFunc.New()
  hash.Write(data)
  digest := hash.Sum(nil)
  hex := fmt.Sprintf("%x", digest)
  ret := ""
  for len(hex) > 0 {
    if len(ret) > 0 {
      ret += ":"
    }
    todo := 2
    if len(hex) < todo {
      todo = len(hex)
    }
    ret += hex[:todo]
    hex = hex[todo:]
  }
  return ret
}
func isHex(c rune) (value byte, ok bool) {
  switch {
  case c >= '0' && c <= '9':
    return byte(c) - '0', true
  case c >= 'a' && c <= 'f':
    return byte(c) - 'a' + 10, true
  case c >= 'A' && c <= 'F':
    return byte(c) - 'A' + 10, true
  }
  return 0, false
}
func appendRune(out []byte, r rune) []byte {
  if r < 128 {
    return append(out, byte(r))
  }
  var buf [utf8.UTFMax]byte
  n := utf8.EncodeRune(buf[:], r)
  return append(out, buf[:n]...)
}
// unescapeLabel unescapes "\xab" style hex-escapes.
func unescapeLabel(escaped string) string {
  var out []byte
  var last rune
  var value byte
  state := 0
  for _, r := range escaped {
    switch state {
    case 0:
      if r == '\\' {
        state++
        continue
      }
    case 1:
      if r == 'x' {
        state++
        continue
      }
      out = append(out, '\\')
    case 2:
      if v, ok := isHex(r); ok {
        value = v
        last = r
        state++
        continue
      } else {
        out = append(out, '\\', 'x')
      }
    case 3:
      if v, ok := isHex(r); ok {
        value <<= 4
        value += v
        out = append(out, byte(value))
        state = 0
        continue
      } else {
        out = append(out, '\\', 'x')
        out = appendRune(out, last)
      }
    }
    state = 0
    out = appendRune(out, r)
  }
  switch state {
  case 3:
    out = append(out, '\\', 'x')
    out = appendRune(out, last)
  case 2:
    out = append(out, '\\', 'x')
  case 1:
    out = append(out, '\\')
  }
  return string(out)
}