vendor: Update github.com/gogo/protobuf

Also tweaks the proto definitions:

 - [packed=false] on the block_indexes field to retain compat with
   v0.14.16 and earlier.

 - Uses the vendored protobuf package in include paths.

And, "build.go setup" will install the vendored protoc-gen-gogofast.
This should ensure that a proto rebuild isn't so dependent on whatever
version of the compiler and package the developer has installed...

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3864
This commit is contained in:
Jakob Borg
2017-01-03 00:16:21 +00:00
committed by Audrius Butkevicius
parent 4fb9c143ac
commit 987718baf8
603 changed files with 340684 additions and 62506 deletions

View File

@@ -41,18 +41,17 @@ package jsonpb
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/gogo/protobuf/proto"
)
var (
byteArrayType = reflect.TypeOf([]byte{})
"github.com/gogo/protobuf/types"
)
// Marshaler is a configurable object for converting between
@@ -69,12 +68,15 @@ type Marshaler struct {
// value, and for newlines to be appear between fields and array
// elements.
Indent string
// Whether to use the original (.proto) name for fields.
OrigName bool
}
// Marshal marshals a protocol buffer into JSON.
func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error {
writer := &errWriter{writer: out}
return m.marshalObject(writer, pb, "")
return m.marshalObject(writer, pb, "", "")
}
// MarshalToString converts a protocol buffer object to JSON string.
@@ -93,15 +95,83 @@ func (s int32Slice) Len() int { return len(s) }
func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] }
func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
type isWkt interface {
XXX_WellKnownType() string
}
// marshalObject writes a struct to the Writer.
func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent string) error {
func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error {
s := reflect.ValueOf(v).Elem()
// Handle well-known types.
if wkt, ok := v.(isWkt); ok {
switch wkt.XXX_WellKnownType() {
case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
"Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
// "Wrappers use the same representation in JSON
// as the wrapped primitive type, ..."
sprop := proto.GetProperties(s.Type())
return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent)
case "Any":
// Any is a bit more involved.
return m.marshalAny(out, v, indent)
case "Duration":
// "Generated output always contains 3, 6, or 9 fractional digits,
// depending on required precision."
s, ns := s.Field(0).Int(), s.Field(1).Int()
d := time.Duration(s)*time.Second + time.Duration(ns)*time.Nanosecond
x := fmt.Sprintf("%.9f", d.Seconds())
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
out.write(`"`)
out.write(x)
out.write(`s"`)
return out.err
case "Struct":
// Let marshalValue handle the `fields` map.
// TODO: pass the correct Properties if needed.
return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent)
case "Timestamp":
// "RFC 3339, where generated output will always be Z-normalized
// and uses 3, 6 or 9 fractional digits."
s, ns := s.Field(0).Int(), s.Field(1).Int()
t := time.Unix(s, ns).UTC()
// time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
x := t.Format("2006-01-02T15:04:05.000000000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
out.write(`"`)
out.write(x)
out.write(`Z"`)
return out.err
case "Value":
// Value has a single oneof.
kind := s.Field(0)
if kind.IsNil() {
// "absence of any variant indicates an error"
return errors.New("nil Value")
}
// oneof -> *T -> T -> T.F
x := kind.Elem().Elem().Field(0)
// TODO: pass the correct Properties if needed.
return m.marshalValue(out, &proto.Properties{}, x, indent)
}
}
out.write("{")
if m.Indent != "" {
out.write("\n")
}
s := reflect.ValueOf(v).Elem()
firstField := true
if typeURL != "" {
if err := m.marshalTypeURL(out, indent, typeURL); err != nil {
return err
}
firstField = false
}
for i := 0; i < s.NumField(); i++ {
value := s.Field(i)
valueField := s.Type().Field(i)
@@ -149,7 +219,7 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent string
value = sv.Field(0)
valueField = sv.Type().Field(0)
}
prop := jsonProperties(valueField)
prop := jsonProperties(valueField, m.OrigName)
if !firstField {
m.writeSep(out)
}
@@ -176,12 +246,14 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent string
}
// Handle proto2 extensions.
if ep, ok := v.(extendableProto); ok {
if ep, ok := v.(proto.Message); ok {
extensions := proto.RegisteredExtensions(v)
extensionMap := ep.ExtensionMap()
// Sort extensions for stable output.
ids := make([]int32, 0, len(extensionMap))
for id := range extensionMap {
ids := make([]int32, 0, len(extensions))
for id, desc := range extensions {
if !proto.HasExtension(ep, desc) {
continue
}
ids = append(ids, id)
}
sort.Sort(int32Slice(ids))
@@ -198,7 +270,7 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent string
value := reflect.ValueOf(ext)
var prop proto.Properties
prop.Parse(desc.Tag)
prop.OrigName = fmt.Sprintf("[%s]", desc.Name)
prop.JSONName = fmt.Sprintf("[%s]", desc.Name)
if !firstField {
m.writeSep(out)
}
@@ -226,6 +298,76 @@ func (m *Marshaler) writeSep(out *errWriter) {
}
}
func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error {
// "If the Any contains a value that has a special JSON mapping,
// it will be converted as follows: {"@type": xxx, "value": yyy}.
// Otherwise, the value will be converted into a JSON object,
// and the "@type" field will be inserted to indicate the actual data type."
v := reflect.ValueOf(any).Elem()
turl := v.Field(0).String()
val := v.Field(1).Bytes()
// Only the part of type_url after the last slash is relevant.
mname := turl
if slash := strings.LastIndex(mname, "/"); slash >= 0 {
mname = mname[slash+1:]
}
mt := proto.MessageType(mname)
if mt == nil {
return fmt.Errorf("unknown message type %q", mname)
}
msg := reflect.New(mt.Elem()).Interface().(proto.Message)
if err := proto.Unmarshal(val, msg); err != nil {
return err
}
if _, ok := msg.(isWkt); ok {
out.write("{")
if m.Indent != "" {
out.write("\n")
}
if err := m.marshalTypeURL(out, indent, turl); err != nil {
return err
}
m.writeSep(out)
if m.Indent != "" {
out.write(indent)
out.write(m.Indent)
out.write(`"value": `)
} else {
out.write(`"value":`)
}
if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil {
return err
}
if m.Indent != "" {
out.write("\n")
out.write(indent)
}
out.write("}")
return out.err
}
return m.marshalObject(out, msg, indent, turl)
}
func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error {
if m.Indent != "" {
out.write(indent)
out.write(m.Indent)
}
out.write(`"@type":`)
if m.Indent != "" {
out.write(" ")
}
b, err := json.Marshal(typeURL)
if err != nil {
return err
}
out.write(string(b))
return out.err
}
// marshalField writes field description and value to the Writer.
func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
if m.Indent != "" {
@@ -233,7 +375,7 @@ func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v refle
out.write(m.Indent)
}
out.write(`"`)
out.write(prop.OrigName)
out.write(prop.JSONName)
out.write(`":`)
if m.Indent != "" {
out.write(" ")
@@ -250,7 +392,7 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
v = reflect.Indirect(v)
// Handle repeated elements.
if v.Type() != byteArrayType && v.Kind() == reflect.Slice {
if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
out.write("[")
comma := ""
for i := 0; i < v.Len(); i++ {
@@ -262,7 +404,9 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
out.write(m.Indent)
out.write(m.Indent)
}
m.marshalValue(out, prop, sliceVal, indent+m.Indent)
if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil {
return err
}
comma = ","
}
if m.Indent != "" {
@@ -274,6 +418,29 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
return out.err
}
// Handle well-known types.
// Most are handled up in marshalObject (because 99% are messages).
if wkt, ok := v.Interface().(isWkt); ok {
switch wkt.XXX_WellKnownType() {
case "NullValue":
out.write("null")
return out.err
}
}
if t, ok := v.Interface().(time.Time); ok {
ts, err := types.TimestampProto(t)
if err != nil {
return err
}
return m.marshalValue(out, prop, reflect.ValueOf(ts), indent)
}
if d, ok := v.Interface().(time.Duration); ok {
dur := types.DurationProto(d)
return m.marshalValue(out, prop, reflect.ValueOf(dur), indent)
}
// Handle enumerations.
if !m.EnumsAsInts && prop.Enum != "" {
// Unknown enum values will are stringified by the proto library as their
@@ -338,7 +505,7 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
}
pm = i.Convert(t).Interface().(proto.Message)
}
return m.marshalObject(out, pm, indent+m.Indent)
return m.marshalObject(out, pm, indent+m.Indent, "")
}
// Handle maps.
@@ -408,32 +575,166 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
return out.err
}
// Unmarshaler is a configurable object for converting from a JSON
// representation to a protocol buffer object.
type Unmarshaler struct {
// Whether to allow messages to contain unknown fields, as opposed to
// failing to unmarshal.
AllowUnknownFields bool
}
// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
// This function is lenient and will decode any options permutations of the
// related Marshaler.
func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
inputValue := json.RawMessage{}
if err := dec.Decode(&inputValue); err != nil {
return err
}
return u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil)
}
// Unmarshal unmarshals a JSON object stream into a protocol
// buffer. This function is lenient and will decode any options
// permutations of the related Marshaler.
func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error {
dec := json.NewDecoder(r)
return u.UnmarshalNext(dec, pb)
}
// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
// This function is lenient and will decode any options permutations of the
// related Marshaler.
func UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
return new(Unmarshaler).UnmarshalNext(dec, pb)
}
// Unmarshal unmarshals a JSON object stream into a protocol
// buffer. This function is lenient and will decode any options
// permutations of the related Marshaler.
func Unmarshal(r io.Reader, pb proto.Message) error {
inputValue := json.RawMessage{}
if err := json.NewDecoder(r).Decode(&inputValue); err != nil {
return err
}
return unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue)
return new(Unmarshaler).Unmarshal(r, pb)
}
// UnmarshalString will populate the fields of a protocol buffer based
// on a JSON string. This function is lenient and will decode any options
// permutations of the related Marshaler.
func UnmarshalString(str string, pb proto.Message) error {
return Unmarshal(strings.NewReader(str), pb)
return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb)
}
// unmarshalValue converts/copies a value into the target.
func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
// prop may be nil.
func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error {
targetType := target.Type()
// Allocate memory for pointer fields.
if targetType.Kind() == reflect.Ptr {
target.Set(reflect.New(targetType.Elem()))
return unmarshalValue(target.Elem(), inputValue)
return u.unmarshalValue(target.Elem(), inputValue, prop)
}
// Handle well-known types.
if wkt, ok := target.Addr().Interface().(isWkt); ok {
switch wkt.XXX_WellKnownType() {
case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
"Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
// "Wrappers use the same representation in JSON
// as the wrapped primitive type, except that null is allowed."
// encoding/json will turn JSON `null` into Go `nil`,
// so we don't have to do any extra work.
return u.unmarshalValue(target.Field(0), inputValue, prop)
case "Any":
return fmt.Errorf("unmarshaling Any not supported yet")
case "Duration":
ivStr := string(inputValue)
if ivStr == "null" {
target.Field(0).SetInt(0)
target.Field(1).SetInt(0)
return nil
}
unq, err := strconv.Unquote(ivStr)
if err != nil {
return err
}
d, err := time.ParseDuration(unq)
if err != nil {
return fmt.Errorf("bad Duration: %v", err)
}
ns := d.Nanoseconds()
s := ns / 1e9
ns %= 1e9
target.Field(0).SetInt(s)
target.Field(1).SetInt(ns)
return nil
case "Timestamp":
ivStr := string(inputValue)
if ivStr == "null" {
target.Field(0).SetInt(0)
target.Field(1).SetInt(0)
return nil
}
unq, err := strconv.Unquote(ivStr)
if err != nil {
return err
}
t, err := time.Parse(time.RFC3339Nano, unq)
if err != nil {
return fmt.Errorf("bad Timestamp: %v", err)
}
target.Field(0).SetInt(int64(t.Unix()))
target.Field(1).SetInt(int64(t.Nanosecond()))
return nil
}
}
if t, ok := target.Addr().Interface().(*time.Time); ok {
ts := &types.Timestamp{}
if err := u.unmarshalValue(reflect.ValueOf(ts).Elem(), inputValue, prop); err != nil {
return err
}
tt, err := types.TimestampFromProto(ts)
if err != nil {
return err
}
*t = tt
return nil
}
if d, ok := target.Addr().Interface().(*time.Duration); ok {
dur := &types.Duration{}
if err := u.unmarshalValue(reflect.ValueOf(dur).Elem(), inputValue, prop); err != nil {
return err
}
dd, err := types.DurationFromProto(dur)
if err != nil {
return err
}
*d = dd
return nil
}
// Handle enums, which have an underlying type of int32,
// and may appear as strings.
// The case of an enum appearing as a number is handled
// at the bottom of this function.
if inputValue[0] == '"' && prop != nil && prop.Enum != "" {
vmap := proto.EnumValueMap(prop.Enum)
// Don't need to do unquoting; valid enum names
// are from a limited character set.
s := inputValue[1 : len(inputValue)-1]
n, ok := vmap[string(s)]
if !ok {
return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum)
}
if target.Kind() == reflect.Ptr { // proto2
target.Set(reflect.New(targetType.Elem()))
target = target.Elem()
}
target.SetInt(int64(n))
return nil
}
// Handle nested messages.
@@ -443,59 +744,58 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
return err
}
consumeField := func(prop *proto.Properties) (json.RawMessage, bool) {
// Be liberal in what names we accept; both orig_name and camelName are okay.
fieldNames := acceptedJSONFieldNames(prop)
vOrig, okOrig := jsonFields[fieldNames.orig]
vCamel, okCamel := jsonFields[fieldNames.camel]
if !okOrig && !okCamel {
return nil, false
}
// If, for some reason, both are present in the data, favour the camelName.
var raw json.RawMessage
if okOrig {
raw = vOrig
delete(jsonFields, fieldNames.orig)
}
if okCamel {
raw = vCamel
delete(jsonFields, fieldNames.camel)
}
return raw, true
}
sprops := proto.GetProperties(targetType)
for i := 0; i < target.NumField(); i++ {
ft := target.Type().Field(i)
if strings.HasPrefix(ft.Name, "XXX_") {
continue
}
fieldName := jsonProperties(ft).OrigName
valueForField, ok := jsonFields[fieldName]
valueForField, ok := consumeField(sprops.Prop[i])
if !ok {
continue
}
delete(jsonFields, fieldName)
// Handle enums, which have an underlying type of int32,
// and may appear as strings. We do this while handling
// the struct so we have access to the enum info.
// The case of an enum appearing as a number is handled
// by the recursive call to unmarshalValue.
if enum := sprops.Prop[i].Enum; valueForField[0] == '"' && enum != "" {
vmap := proto.EnumValueMap(enum)
// Don't need to do unquoting; valid enum names
// are from a limited character set.
s := valueForField[1 : len(valueForField)-1]
n, ok := vmap[string(s)]
if !ok {
return fmt.Errorf("unknown value %q for enum %s", s, enum)
}
f := target.Field(i)
if f.Kind() == reflect.Ptr { // proto2
f.Set(reflect.New(f.Type().Elem()))
f = f.Elem()
}
f.SetInt(int64(n))
continue
}
if err := unmarshalValue(target.Field(i), valueForField); err != nil {
if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil {
return err
}
}
// Check for any oneof fields.
for fname, raw := range jsonFields {
if oop, ok := sprops.OneofTypes[fname]; ok {
if len(jsonFields) > 0 {
for _, oop := range sprops.OneofTypes {
raw, ok := consumeField(oop.Prop)
if !ok {
continue
}
nv := reflect.New(oop.Type.Elem())
target.Field(oop.Field).Set(nv)
if err := unmarshalValue(nv.Elem().Field(0), raw); err != nil {
if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil {
return err
}
delete(jsonFields, fname)
}
}
if len(jsonFields) > 0 {
if !u.AllowUnknownFields && len(jsonFields) > 0 {
// Pick any field to be the scapegoat.
var f string
for fname := range jsonFields {
@@ -507,8 +807,33 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
return nil
}
// Handle arrays (which aren't encoded bytes)
if targetType != byteArrayType && targetType.Kind() == reflect.Slice {
// Handle arrays
if targetType.Kind() == reflect.Slice {
if targetType.Elem().Kind() == reflect.Uint8 {
outRef := reflect.New(targetType)
outVal := outRef.Interface()
//CustomType with underlying type []byte
if _, ok := outVal.(interface {
UnmarshalJSON([]byte) error
}); ok {
if err := json.Unmarshal(inputValue, outVal); err != nil {
return err
}
target.Set(outRef.Elem())
return nil
}
// Special case for encoded bytes. Pre-go1.5 doesn't support unmarshalling
// strings into aliased []byte types.
// https://github.com/golang/go/commit/4302fd0409da5e4f1d71471a6770dacdc3301197
// https://github.com/golang/go/commit/c60707b14d6be26bf4213114d13070bff00d0b0a
var out []byte
if err := json.Unmarshal(inputValue, &out); err != nil {
return err
}
target.SetBytes(out)
return nil
}
var slc []json.RawMessage
if err := json.Unmarshal(inputValue, &slc); err != nil {
return err
@@ -516,7 +841,7 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
len := len(slc)
target.Set(reflect.MakeSlice(targetType, len, len))
for i := 0; i < len; i++ {
if err := unmarshalValue(target.Index(i), slc[i]); err != nil {
if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil {
return err
}
}
@@ -530,6 +855,13 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
return err
}
target.Set(reflect.MakeMap(targetType))
var keyprop, valprop *proto.Properties
if prop != nil {
// These could still be nil if the protobuf metadata is broken somehow.
// TODO: This won't work because the fields are unexported.
// We should probably just reparse them.
//keyprop, valprop = prop.mkeyprop, prop.mvalprop
}
for ks, raw := range mp {
// Unmarshal map key. The core json library already decoded the key into a
// string, so we handle that specially. Other types were quoted post-serialization.
@@ -538,7 +870,7 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
k = reflect.ValueOf(ks)
} else {
k = reflect.New(targetType.Key()).Elem()
if err := unmarshalValue(k, json.RawMessage(ks)); err != nil {
if err := u.unmarshalValue(k, json.RawMessage(ks), keyprop); err != nil {
return err
}
}
@@ -549,7 +881,7 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
// Unmarshal map value.
v := reflect.New(targetType.Elem()).Elem()
if err := unmarshalValue(v, raw); err != nil {
if err := u.unmarshalValue(v, raw, valprop); err != nil {
return err
}
target.SetMapIndex(k, v)
@@ -568,18 +900,26 @@ func unmarshalValue(target reflect.Value, inputValue json.RawMessage) error {
return json.Unmarshal(inputValue, target.Addr().Interface())
}
// jsonProperties returns parsed proto.Properties for the field.
func jsonProperties(f reflect.StructField) *proto.Properties {
// jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute.
func jsonProperties(f reflect.StructField, origName bool) *proto.Properties {
var prop proto.Properties
prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f)
if origName || prop.JSONName == "" {
prop.JSONName = prop.OrigName
}
return &prop
}
// extendableProto is an interface implemented by any protocol buffer that may be extended.
type extendableProto interface {
proto.Message
ExtensionRangeArray() []proto.ExtensionRange
ExtensionMap() map[int32]proto.Extension
type fieldNames struct {
orig, camel string
}
func acceptedJSONFieldNames(prop *proto.Properties) fieldNames {
opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName}
if prop.JSONName != "" {
opts.camel = prop.JSONName
}
return opts
}
// Writer wrapper inspired by https://blog.golang.org/errors-are-values
@@ -599,10 +939,21 @@ func (w *errWriter) write(str string) {
// The easiest way to sort them in some deterministic order is to use fmt.
// If this turns out to be inefficient we can always consider other options,
// such as doing a Schwartzian transform.
//
// Numeric keys are sorted in numeric order per
// https://developers.google.com/protocol-buffers/docs/proto#maps.
type mapKeys []reflect.Value
func (s mapKeys) Len() int { return len(s) }
func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s mapKeys) Less(i, j int) bool {
if k := s[i].Kind(); k == s[j].Kind() {
switch k {
case reflect.Int32, reflect.Int64:
return s[i].Int() < s[j].Int()
case reflect.Uint32, reflect.Uint64:
return s[i].Uint() < s[j].Uint()
}
}
return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface())
}