parse.js

var tokenize = require('./tokenize')
var MAX_RANGE = 0x1FFFFFFF

// "Only repeated fields of primitive numeric types (types which use the varint, 32-bit, or 64-bit wire types) can be declared "packed"."
// https://developers.google.com/protocol-buffers/docs/encoding#optional
var PACKABLE_TYPES = [
  // varint wire types
  'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64', 'bool',
  // + ENUMS
  // 64-bit wire types
  'fixed64', 'sfixed64', 'double',
  // 32-bit wire types
  'fixed32', 'sfixed32', 'float'
]

var onfieldoptionvalue = function (tokens) {
  var value = tokens.shift()
  if (value !== '{') {
    return value
  }
  value = {}
  var field = ''
  while (tokens.length) {
    switch (tokens[0]) {
      case '}':
        tokens.shift()
        return value
      case ':':
        tokens.shift()
        value[field] = onfieldoptionvalue(tokens)
        break
      default:
        field = tokens.shift()
    }
  }
}

var onfieldoptions = function (tokens) {
  var opts = {}

  while (tokens.length) {
    switch (tokens[0]) {
      case '[':
      case ',': {
        tokens.shift()
        var name = tokens.shift()
        if (name === '(') { // handling [(A) = B]
          name = tokens.shift()
          tokens.shift() // remove the end of bracket
        }
        var field = []
        if (tokens[0][0] === '.') {
          field = tokens[0].substr(1).split('.')
          tokens.shift()
        }
        if (tokens[0] !== '=') throw new Error('Unexpected token in field options: ' + tokens[0])
        tokens.shift()
        if (tokens[0] === ']') throw new Error('Unexpected ] in field option')

        // for option (A).b.c
        // path will be ['A', 'b'] and lastFieldName 'c'
        var path = [name].concat(field)
        var lastFieldName = path.pop()

        // opt references opts.A.b
        var opt = path.reduce(function (opt, n, index) {
          if (opt[n] == null) {
            opt[n] = {}
          }
          return opt[n]
        }, opts)

        // now set opt['c'] that references opts.A.b['c']
        opt[lastFieldName] = onfieldoptionvalue(tokens)
        break
      }
      case ']':
        tokens.shift()
        return opts

      default:
        throw new Error('Unexpected token in field options: ' + tokens[0])
    }
  }

  throw new Error('No closing tag for field options')
}

var onfield = function (tokens) {
  var field = {
    name: null,
    type: null,
    tag: -1,
    map: null,
    oneof: null,
    required: false,
    repeated: false,
    options: {}
  }

  while (tokens.length) {
    switch (tokens[0]) {
      case '=':
        tokens.shift()
        field.tag = Number(tokens.shift())
        break

      case 'map':
        field.type = 'map'
        field.map = { from: null, to: null }
        tokens.shift()
        if (tokens[0] !== '<') throw new Error('Unexpected token in map type: ' + tokens[0])
        tokens.shift()
        field.map.from = tokens.shift()
        if (tokens[0] !== ',') throw new Error('Unexpected token in map type: ' + tokens[0])
        tokens.shift()
        field.map.to = tokens.shift()
        if (tokens[0] !== '>') throw new Error('Unexpected token in map type: ' + tokens[0])
        tokens.shift()
        field.name = tokens.shift()
        break

      case 'repeated':
      case 'required':
      case 'optional':
        var t = tokens.shift()
        field.required = t === 'required'
        field.repeated = t === 'repeated'
        field.type = tokens.shift()
        field.name = tokens.shift()
        break

      case '[':
        field.options = onfieldoptions(tokens)
        break

      case ';':
        if (field.name === null) throw new Error('Missing field name')
        if (field.type === null) throw new Error('Missing type in message field: ' + field.name)
        if (field.tag === -1) throw new Error('Missing tag number in message field: ' + field.name)
        tokens.shift()
        return field

      default:
        throw new Error('Unexpected token in message field: ' + tokens[0])
    }
  }

  throw new Error('No ; found for message field')
}

var onmessagebody = function (tokens) {
  var body = {
    enums: [],
    options: {},
    messages: [],
    fields: [],
    extends: [],
    extensions: null
  }

  while (tokens.length) {
    switch (tokens[0]) {
      case 'map':
      case 'repeated':
      case 'optional':
      case 'required':
        body.fields.push(onfield(tokens))
        break

      case 'enum':
        body.enums.push(onenum(tokens))
        break

      case 'message':
        body.messages.push(onmessage(tokens))
        break

      case 'extensions':
        body.extensions = onextensions(tokens)
        break

      case 'oneof':
        tokens.shift()
        var name = tokens.shift()
        if (tokens[0] !== '{') throw new Error('Unexpected token in oneof: ' + tokens[0])
        tokens.shift()
        while (tokens[0] !== '}') {
          tokens.unshift('optional')
          var field = onfield(tokens)
          field.oneof = name
          body.fields.push(field)
        }
        tokens.shift()
        break

      case 'extend':
        body.extends.push(onextend(tokens))
        break

      case ';':
        tokens.shift()
        break

      case 'reserved':
        tokens.shift()
        while (tokens[0] !== ';') {
          tokens.shift()
        }
        break

      case 'option':
        var opt = onoption(tokens)
        if (body.options[opt.name] !== undefined) throw new Error('Duplicate option ' + opt.name)
        body.options[opt.name] = opt.value
        break

      default:
        // proto3 does not require the use of optional/required, assumed as optional
        // "singular: a well-formed message can have zero or one of this field (but not more than one)."
        // https://developers.google.com/protocol-buffers/docs/proto3#specifying-field-rules
        tokens.unshift('optional')
        body.fields.push(onfield(tokens))
    }
  }

  return body
}

var onextend = function (tokens) {
  var out = {
    name: tokens[1],
    message: onmessage(tokens)
  }
  return out
}

var onextensions = function (tokens) {
  tokens.shift()
  var from = Number(tokens.shift())
  if (isNaN(from)) throw new Error('Invalid from in extensions definition')
  if (tokens.shift() !== 'to') throw new Error("Expected keyword 'to' in extensions definition")
  var to = tokens.shift()
  if (to === 'max') to = MAX_RANGE
  to = Number(to)
  if (isNaN(to)) throw new Error('Invalid to in extensions definition')
  if (tokens.shift() !== ';') throw new Error('Missing ; in extensions definition')
  return { from: from, to: to }
}
var onmessage = function (tokens) {
  tokens.shift()

  var lvl = 1
  var body = []
  var msg = {
    name: tokens.shift(),
    options: {},
    enums: [],
    extends: [],
    messages: [],
    fields: []
  }

  if (tokens[0] !== '{') throw new Error('Expected { but found ' + tokens[0])
  tokens.shift()

  while (tokens.length) {
    if (tokens[0] === '{') lvl++
    else if (tokens[0] === '}') lvl--

    if (!lvl) {
      tokens.shift()
      body = onmessagebody(body)
      msg.enums = body.enums
      msg.messages = body.messages
      msg.fields = body.fields
      msg.extends = body.extends
      msg.extensions = body.extensions
      msg.options = body.options
      return msg
    }

    body.push(tokens.shift())
  }

  if (lvl) throw new Error('No closing tag for message')
}

var onpackagename = function (tokens) {
  tokens.shift()
  var name = tokens.shift()
  if (tokens[0] !== ';') throw new Error('Expected ; but found ' + tokens[0])
  tokens.shift()
  return name
}

var onsyntaxversion = function (tokens) {
  tokens.shift()

  if (tokens[0] !== '=') throw new Error('Expected = but found ' + tokens[0])
  tokens.shift()

  var version = tokens.shift()
  switch (version) {
    case '"proto2"':
      version = 2
      break

    case '"proto3"':
      version = 3
      break

    default:
      throw new Error('Expected protobuf syntax version but found ' + version)
  }

  if (tokens[0] !== ';') throw new Error('Expected ; but found ' + tokens[0])
  tokens.shift()

  return version
}

var onenumvalue = function (tokens) {
  if (tokens.length < 4) throw new Error('Invalid enum value: ' + tokens.slice(0, 3).join(' '))
  if (tokens[0] === 'reserved') {
    tokens.shift()
    while (tokens[0] !== ';') {
      tokens.shift()
    }
    tokens.shift()
    return null
  }
  if (tokens[1] !== '=') throw new Error('Expected = but found ' + tokens[1])
  if (tokens[3] !== ';' && tokens[3] !== '[') throw new Error('Expected ; or [ but found ' + tokens[1])

  var name = tokens.shift()
  tokens.shift()
  var val = {
    value: null,
    options: {}
  }
  val.value = Number(tokens.shift())
  if (tokens[0] === '[') {
    val.options = onfieldoptions(tokens)
  }
  tokens.shift() // expecting the semicolon here

  return {
    name: name,
    val: val
  }
}

var onenum = function (tokens) {
  tokens.shift()
  var options = {}
  var e = {
    name: tokens.shift(),
    values: {},
    options: {}
  }

  if (tokens[0] !== '{') throw new Error('Expected { but found ' + tokens[0])
  tokens.shift()

  while (tokens.length) {
    if (tokens[0] === '}') {
      tokens.shift()
      // there goes optional semicolon after the enclosing "}"
      if (tokens[0] === ';') tokens.shift()
      return e
    }
    if (tokens[0] === 'option') {
      options = onoption(tokens)
      e.options[options.name] = options.value
      continue
    }
    var val = onenumvalue(tokens)
    if (val !== null) {
      e.values[val.name] = val.val
    }
  }

  throw new Error('No closing tag for enum')
}

var onoption = function (tokens) {
  var name = null
  var value = null

  var parse = function (value) {
    if (value === 'true') return true
    if (value === 'false') return false
    return value.replace(/^"+|"+$/gm, '')
  }

  while (tokens.length) {
    if (tokens[0] === ';') {
      tokens.shift()
      return { name: name, value: value }
    }
    switch (tokens[0]) {
      case 'option':
        tokens.shift()

        var hasBracket = tokens[0] === '('
        if (hasBracket) tokens.shift()

        name = tokens.shift()

        if (hasBracket) {
          if (tokens[0] !== ')') throw new Error('Expected ) but found ' + tokens[0])
          tokens.shift()
        }

        if (tokens[0][0] === '.') {
          name += tokens.shift()
        }

        break

      case '=':
        tokens.shift()
        if (name === null) throw new Error('Expected key for option with value: ' + tokens[0])
        value = parse(tokens.shift())

        if (name === 'optimize_for' && !/^(SPEED|CODE_SIZE|LITE_RUNTIME)$/.test(value)) {
          throw new Error('Unexpected value for option optimize_for: ' + value)
        } else if (value === '{') {
          // option foo = {bar: baz}
          value = onoptionMap(tokens)
        }
        break

      default:
        throw new Error('Unexpected token in option: ' + tokens[0])
    }
  }
}

var onoptionMap = function (tokens) {
  var parse = function (value) {
    if (value === 'true') return true
    if (value === 'false') return false
    return value.replace(/^"+|"+$/gm, '')
  }

  var map = {}

  while (tokens.length) {
    if (tokens[0] === '}') {
      tokens.shift()
      return map
    }

    var hasBracket = tokens[0] === '('
    if (hasBracket) tokens.shift()

    var key = tokens.shift()
    if (hasBracket) {
      if (tokens[0] !== ')') throw new Error('Expected ) but found ' + tokens[0])
      tokens.shift()
    }

    var value = null

    switch (tokens[0]) {
      case ':':
        if (map[key] !== undefined) throw new Error('Duplicate option map key ' + key)

        tokens.shift()

        value = parse(tokens.shift())

        if (value === '{') {
          // option foo = {bar: baz}
          value = onoptionMap(tokens)
        }

        map[key] = value

        if (tokens[0] === ';') {
          tokens.shift()
        }
        break

      case '{':
        tokens.shift()
        value = onoptionMap(tokens)

        if (map[key] === undefined) map[key] = []
        if (!Array.isArray(map[key])) throw new Error('Duplicate option map key ' + key)

        map[key].push(value)
        break

      default:
        throw new Error('Unexpected token in option map: ' + tokens[0])
    }
  }

  throw new Error('No closing tag for option map')
}

var onimport = function (tokens) {
  tokens.shift()
  var file = tokens.shift().replace(/^"+|"+$/gm, '')

  if (tokens[0] !== ';') throw new Error('Unexpected token: ' + tokens[0] + '. Expected ";"')

  tokens.shift()
  return file
}

var onservice = function (tokens) {
  tokens.shift()

  var service = {
    name: tokens.shift(),
    methods: [],
    options: {}
  }

  if (tokens[0] !== '{') throw new Error('Expected { but found ' + tokens[0])
  tokens.shift()

  while (tokens.length) {
    if (tokens[0] === '}') {
      tokens.shift()
      // there goes optional semicolon after the enclosing "}"
      if (tokens[0] === ';') tokens.shift()
      return service
    }

    switch (tokens[0]) {
      case 'option':
        var opt = onoption(tokens)
        if (service.options[opt.name] !== undefined) throw new Error('Duplicate option ' + opt.name)
        service.options[opt.name] = opt.value
        break
      case 'rpc':
        service.methods.push(onrpc(tokens))
        break
      default:
        throw new Error('Unexpected token in service: ' + tokens[0])
    }
  }

  throw new Error('No closing tag for service')
}

var onrpc = function (tokens) {
  tokens.shift()

  var rpc = {
    name: tokens.shift(),
    input_type: null,
    output_type: null,
    client_streaming: false,
    server_streaming: false,
    options: {}
  }

  if (tokens[0] !== '(') throw new Error('Expected ( but found ' + tokens[0])
  tokens.shift()

  if (tokens[0] === 'stream') {
    tokens.shift()
    rpc.client_streaming = true
  }

  rpc.input_type = tokens.shift()

  if (tokens[0] !== ')') throw new Error('Expected ) but found ' + tokens[0])
  tokens.shift()

  if (tokens[0] !== 'returns') throw new Error('Expected returns but found ' + tokens[0])
  tokens.shift()

  if (tokens[0] !== '(') throw new Error('Expected ( but found ' + tokens[0])
  tokens.shift()

  if (tokens[0] === 'stream') {
    tokens.shift()
    rpc.server_streaming = true
  }

  rpc.output_type = tokens.shift()

  if (tokens[0] !== ')') throw new Error('Expected ) but found ' + tokens[0])
  tokens.shift()

  if (tokens[0] === ';') {
    tokens.shift()
    return rpc
  }

  if (tokens[0] !== '{') throw new Error('Expected { but found ' + tokens[0])
  tokens.shift()

  while (tokens.length) {
    if (tokens[0] === '}') {
      tokens.shift()
      // there goes optional semicolon after the enclosing "}"
      if (tokens[0] === ';') tokens.shift()
      return rpc
    }

    if (tokens[0] === 'option') {
      var opt = onoption(tokens)
      if (rpc.options[opt.name] !== undefined) throw new Error('Duplicate option ' + opt.name)
      rpc.options[opt.name] = opt.value
    } else {
      throw new Error('Unexpected token in rpc options: ' + tokens[0])
    }
  }

  throw new Error('No closing tag for rpc')
}

var parse = function (buf) {
  var tokens = tokenize(buf.toString())
  // check for isolated strings in tokens by looking for opening quote
  for (var i = 0; i < tokens.length; i++) {
    if (/^("|')([^'"]*)$/.test(tokens[i])) {
      var j
      if (tokens[i].length === 1) {
        j = i + 1
      } else {
        j = i
      }
      // look ahead for the closing quote and collapse all
      // in-between tokens into a single token
      for (j; j < tokens.length; j++) {
        if (/^[^'"\\]*(?:\\.[^'"\\]*)*("|')$/.test(tokens[j])) {
          tokens = tokens.slice(0, i).concat(tokens.slice(i, j + 1).join('')).concat(tokens.slice(j + 1))
          break
        }
      }
    }
  }
  var schema = {
    syntax: 3,
    package: null,
    imports: [],
    enums: [],
    messages: [],
    options: {},
    extends: []
  }

  var firstline = true

  while (tokens.length) {
    switch (tokens[0]) {
      case 'package':
        schema.package = onpackagename(tokens)
        break

      case 'syntax':
        if (!firstline) throw new Error('Protobuf syntax version should be first thing in file')
        schema.syntax = onsyntaxversion(tokens)
        break

      case 'message':
        schema.messages.push(onmessage(tokens))
        break

      case 'enum':
        schema.enums.push(onenum(tokens))
        break

      case 'option':
        var opt = onoption(tokens)
        if (schema.options[opt.name]) throw new Error('Duplicate option ' + opt.name)
        schema.options[opt.name] = opt.value
        break

      case 'import':
        schema.imports.push(onimport(tokens))
        break

      case 'extend':
        schema.extends.push(onextend(tokens))
        break

      case 'service':
        if (!schema.services) schema.services = []
        schema.services.push(onservice(tokens))
        break

      default:
        throw new Error('Unexpected token: ' + tokens[0])
    }
    firstline = false
  }

  // now iterate over messages and propagate extends
  schema.extends.forEach(function (ext) {
    schema.messages.forEach(function (msg) {
      if (msg.name === ext.name) {
        ext.message.fields.forEach(function (field) {
          if (!msg.extensions || field.tag < msg.extensions.from || field.tag > msg.extensions.to) {
            throw new Error(msg.name + ' does not declare ' + field.tag + ' as an extension number')
          }
          msg.fields.push(field)
        })
      }
    })
  })

  schema.messages.forEach(function (msg) {
    msg.fields.forEach(function (field) {
      var fieldSplit
      var messageName
      var nestedEnumName
      var message

      function enumNameIsFieldType (en) {
        return en.name === field.type
      }

      function enumNameIsNestedEnumName (en) {
        return en.name === nestedEnumName
      }

      if (field.options && field.options.packed === 'true') {
        if (PACKABLE_TYPES.indexOf(field.type) === -1) {
          // let's see if it's an enum
          if (field.type.indexOf('.') === -1) {
            if (msg.enums && msg.enums.some(enumNameIsFieldType)) {
              return
            }
          } else {
            fieldSplit = field.type.split('.')
            if (fieldSplit.length > 2) {
              throw new Error('what is this?')
            }

            messageName = fieldSplit[0]
            nestedEnumName = fieldSplit[1]

            schema.messages.some(function (msg) {
              if (msg.name === messageName) {
                message = msg
                return msg
              }
            })

            if (message && message.enums && message.enums.some(enumNameIsNestedEnumName)) {
              return
            }
          }

          throw new Error(
            'Fields of type ' + field.type + ' cannot be declared [packed=true]. ' +
            'Only repeated fields of primitive numeric types (types which use ' +
            'the varint, 32-bit, or 64-bit wire types) can be declared "packed". ' +
            'See https://developers.google.com/protocol-buffers/docs/encoding#optional'
          )
        }
      }
    })
  })

  return schema
}

module.exports = parse