Encoding Formats

import (
    "google.golang.org/protobuf/encoding/protojson"
    "google.golang.org/protobuf/encoding/prototext"
    "google.golang.org/protobuf/encoding/protowire"
    "google.golang.org/protobuf/encoding/protodelim"
    "google.golang.org/protobuf/proto"
)

// Marshal writes the given proto.Message in JSON format using default options
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func Marshal(m proto.Message) ([]byte, error)

// Unmarshal reads the given []byte into the given proto.Message
// The provided message must be mutable (e.g., a non-nil pointer to a message)
func Unmarshal(b []byte, m proto.Message) error

// Format formats the message as a multiline string
// This function is only intended for human consumption and ignores errors
// Do not depend on the output being stable
func Format(m proto.Message) string

import "google.golang.org/protobuf/encoding/protojson"

msg := &pb.MyMessage{
    Name: "example",
    Value: 42,
}

// Marshal to JSON
jsonData, err := protojson.Marshal(msg)
if err != nil {
    log.Fatal(err)
}
fmt.Println(string(jsonData))

// Unmarshal from JSON
msg2 := &pb.MyMessage{}
err = protojson.Unmarshal(jsonData, msg2)
if err != nil {
    log.Fatal(err)
}

// Format for human reading
formatted := protojson.Format(msg)
fmt.Println(formatted)

// MarshalOptions is a configurable JSON format marshaler
type MarshalOptions struct {
    pragma.NoUnkeyedLiterals

    // Multiline specifies whether the marshaler should format the output in
    // indented-form with every textual element on a new line.
    // If Indent is an empty string, then an arbitrary indent is chosen.
    Multiline bool

    // Indent specifies the set of indentation characters to use in a multiline
    // formatted output such that every entry is preceded by Indent and
    // terminated by a newline. If non-empty, then Multiline is treated as true.
    // Indent can only be composed of space or tab characters.
    Indent string

    // AllowPartial allows messages that have missing required fields to marshal
    // without returning an error. If AllowPartial is false (the default),
    // Marshal will return error if there are any missing required fields.
    AllowPartial bool

    // UseProtoNames uses proto field name instead of lowerCamelCase name in JSON
    // field names.
    UseProtoNames bool

    // UseEnumNumbers emits enum values as numbers.
    UseEnumNumbers bool

    // EmitUnpopulated specifies whether to emit unpopulated fields. It does not
    // emit unpopulated oneof fields or unpopulated extension fields.
    // The JSON value emitted for unpopulated fields are as follows:
    //  ╔═══════╤════════════════════════════╗
    //  ║ JSON  │ Protobuf field             ║
    //  ╠═══════╪════════════════════════════╣
    //  ║ false │ proto3 boolean fields      ║
    //  ║ 0     │ proto3 numeric fields      ║
    //  ║ ""    │ proto3 string/bytes fields ║
    //  ║ null  │ proto2 scalar fields       ║
    //  ║ null  │ message fields             ║
    //  ║ []    │ list fields                ║
    //  ║ {}    │ map fields                 ║
    //  ╚═══════╧════════════════════════════╝
    EmitUnpopulated bool

    // EmitDefaultValues specifies whether to emit default-valued primitive fields,
    // empty lists, and empty maps. The fields affected are as follows:
    //  ╔═══════╤════════════════════════════════════════╗
    //  ║ JSON  │ Protobuf field                         ║
    //  ╠═══════╪════════════════════════════════════════╣
    //  ║ false │ non-optional scalar boolean fields     ║
    //  ║ 0     │ non-optional scalar numeric fields     ║
    //  ║ ""    │ non-optional scalar string/byte fields ║
    //  ║ []    │ empty repeated fields                  ║
    //  ║ {}    │ empty map fields                       ║
    //  ╚═══════╧════════════════════════════════════════╝
    //
    // Behaves similarly to EmitUnpopulated, but does not emit "null"-value fields,
    // i.e. presence-sensing fields that are omitted will remain omitted to preserve
    // presence-sensing.
    // EmitUnpopulated takes precedence over EmitDefaultValues since the former generates
    // a strict superset of the latter.
    EmitDefaultValues bool

    // Resolver is used for looking up types when expanding google.protobuf.Any
    // messages. If nil, this defaults to using protoregistry.GlobalTypes.
    Resolver interface {
        protoregistry.ExtensionTypeResolver
        protoregistry.MessageTypeResolver
    }
}

// Marshal marshals the given proto.Message in the JSON format using options
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error)

// MarshalAppend appends the JSON format encoding of m to b, returning the result
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error)

// Format formats the message as a string
// This method is only intended for human consumption and ignores errors
func (o MarshalOptions) Format(m proto.Message) string

opts := protojson.MarshalOptions{
    Multiline:        true,
    Indent:           "  ",
    UseProtoNames:    true,
    EmitUnpopulated:  true,
}

jsonData, err := opts.Marshal(msg)
if err != nil {
    log.Fatal(err)
}

// UnmarshalOptions is a configurable JSON format parser
type UnmarshalOptions struct {
    pragma.NoUnkeyedLiterals

    // If AllowPartial is set, input for messages that will result in missing
    // required fields will not return an error.
    AllowPartial bool

    // If DiscardUnknown is set, unknown fields and enum name values are ignored.
    DiscardUnknown bool

    // Resolver is used for looking up types when unmarshaling
    // google.protobuf.Any messages or extension fields.
    // If nil, this defaults to using protoregistry.GlobalTypes.
    Resolver interface {
        protoregistry.MessageTypeResolver
        protoregistry.ExtensionTypeResolver
    }

    // RecursionLimit limits how deeply messages may be nested.
    // If zero, a default limit is applied.
    RecursionLimit int
}

// Unmarshal reads the given []byte and populates the given proto.Message using
// options in the UnmarshalOptions object. It will clear the message first
// before setting the fields. If it returns an error, the given message may be
// partially set. The provided message must be mutable (e.g., a non-nil pointer
// to a message)
func (o UnmarshalOptions) Unmarshal(b []byte, m proto.Message) error

opts := protojson.UnmarshalOptions{
    DiscardUnknown: true,
    AllowPartial:   false,
}

msg := &pb.MyMessage{}
err := opts.Unmarshal(jsonData, msg)
if err != nil {
    log.Fatal(err)
}

// Marshal writes the given proto.Message in textproto format using default options
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func Marshal(m proto.Message) ([]byte, error)

// Unmarshal reads the given []byte into the given proto.Message
// The provided message must be mutable (e.g., a non-nil pointer to a message)
func Unmarshal(b []byte, m proto.Message) error

// Format formats the message as a multiline string
// This function is only intended for human consumption and ignores errors
// Do not depend on the output being stable
func Format(m proto.Message) string

import "google.golang.org/protobuf/encoding/prototext"

msg := &pb.MyMessage{
    Name: "example",
    Value: 42,
}

// Marshal to text format
textData, err := prototext.Marshal(msg)
if err != nil {
    log.Fatal(err)
}
fmt.Println(string(textData))

// Unmarshal from text format
msg2 := &pb.MyMessage{}
err = prototext.Unmarshal(textData, msg2)
if err != nil {
    log.Fatal(err)
}

// MarshalOptions is a configurable text format marshaler
type MarshalOptions struct {
    pragma.NoUnkeyedLiterals

    // Multiline specifies whether the marshaler should format the output in
    // indented-form with every textual element on a new line.
    // If Indent is an empty string, then an arbitrary indent is chosen.
    Multiline bool

    // Indent specifies the set of indentation characters to use in a multiline
    // formatted output such that every entry is preceded by Indent and
    // terminated by a newline. If non-empty, then Multiline is treated as true.
    // Indent can only be composed of space or tab characters.
    Indent string

    // EmitASCII specifies whether to format strings and bytes as ASCII only
    // as opposed to using UTF-8 encoding when possible.
    EmitASCII bool

    // AllowPartial allows messages that have missing required fields to marshal
    // without returning an error. If AllowPartial is false (the default),
    // Marshal will return error if there are any missing required fields.
    AllowPartial bool

    // EmitUnknown specifies whether to emit unknown fields in the output.
    // If specified, the unmarshaler may be unable to parse the output.
    // The default is to exclude unknown fields.
    EmitUnknown bool

    // Resolver is used for looking up types when expanding google.protobuf.Any
    // messages. If nil, this defaults to using protoregistry.GlobalTypes.
    Resolver interface {
        protoregistry.ExtensionTypeResolver
        protoregistry.MessageTypeResolver
    }
    // Has unexported fields.
}

// Marshal writes the given proto.Message in textproto format using options in
// MarshalOptions object. Do not depend on the output being stable.
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error)

// MarshalAppend appends the textproto format encoding of m to b, returning the result
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error)

// Format formats the message as a string
// This method is only intended for human consumption and ignores errors
func (o MarshalOptions) Format(m proto.Message) string

// UnmarshalOptions is a configurable textproto format unmarshaler
type UnmarshalOptions struct {
    pragma.NoUnkeyedLiterals

    // AllowPartial accepts input for messages that will result in missing
    // required fields. If AllowPartial is false (the default), Unmarshal will
    // return error if there are any missing required fields.
    AllowPartial bool

    // DiscardUnknown specifies whether to ignore unknown fields when parsing.
    // An unknown field is any field whose field name or field number does not
    // resolve to any known or extension field in the message.
    // By default, unmarshal rejects unknown fields as an error.
    DiscardUnknown bool

    // Resolver is used for looking up types when unmarshaling
    // google.protobuf.Any messages or extension fields.
    // If nil, this defaults to using protoregistry.GlobalTypes.
    Resolver interface {
        protoregistry.MessageTypeResolver
        protoregistry.ExtensionTypeResolver
    }
}

// Unmarshal reads the given []byte and populates the given proto.Message using
// options in the UnmarshalOptions object. The provided message must be mutable
func (o UnmarshalOptions) Unmarshal(b []byte, m proto.Message) error

// Number represents the field number
type Number int32

const (
    MinValidNumber        Number = 1
    FirstReservedNumber   Number = 19000
    LastReservedNumber    Number = 19999
    MaxValidNumber        Number = 1<<29 - 1
    DefaultRecursionLimit        = 10000
)

// IsValid reports whether the field number is semantically valid
func (n Number) IsValid() bool

// Type represents the wire type
type Type int8

const (
    VarintType     Type = 0
    Fixed32Type    Type = 5
    Fixed64Type    Type = 1
    BytesType      Type = 2
    StartGroupType Type = 3
    EndGroupType   Type = 4
)

// AppendVarint appends v to b as a varint-encoded uint64
func AppendVarint(b []byte, v uint64) []byte

// AppendFixed32 appends v to b as a little-endian uint32
func AppendFixed32(b []byte, v uint32) []byte

// AppendFixed64 appends v to b as a little-endian uint64
func AppendFixed64(b []byte, v uint64) []byte

// AppendBytes appends v to b as a length-prefixed bytes value
func AppendBytes(b []byte, v []byte) []byte

// AppendString appends v to b as a length-prefixed bytes value
func AppendString(b []byte, v string) []byte

// AppendTag encodes num and typ as a varint-encoded tag and appends it to b
func AppendTag(b []byte, num Number, typ Type) []byte

// AppendGroup appends v to b as group value, with a trailing end group marker
// The value v must not contain the end marker
func AppendGroup(b []byte, num Number, v []byte) []byte

// ConsumeVarint parses b as a varint-encoded uint64, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeVarint(b []byte) (v uint64, n int)

// ConsumeFixed32 parses b as a little-endian uint32, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeFixed32(b []byte) (v uint32, n int)

// ConsumeFixed64 parses b as a little-endian uint64, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeFixed64(b []byte) (v uint64, n int)

// ConsumeBytes parses b as a length-prefixed bytes value, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeBytes(b []byte) (v []byte, n int)

// ConsumeString parses b as a length-prefixed bytes value, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeString(b []byte) (v string, n int)

// ConsumeTag parses b as a varint-encoded tag, reporting its length
// This returns a negative length upon an error (see ParseError)
func ConsumeTag(b []byte) (Number, Type, int)

// ConsumeField parses an entire field record (both tag and value) and returns
// the field number, the wire type, and the total length
// This returns a negative length upon an error (see ParseError)
// The total length includes the tag header and the end group marker (if the
// field is a group)
func ConsumeField(b []byte) (Number, Type, int)

// ConsumeFieldValue parses a field value and returns its length
// This assumes that the field Number and wire Type have already been parsed
// This returns a negative length upon an error (see ParseError)
// When parsing a group, the length includes the end group marker and the end
// group is verified to match the starting field number
func ConsumeFieldValue(num Number, typ Type, b []byte) (n int)

// ConsumeGroup parses b as a group value until the trailing end group marker,
// and verifies that the end marker matches the provided num
// The value v does not contain the end marker, while the length does contain
// the end marker. This returns a negative length upon an error (see ParseError)
func ConsumeGroup(num Number, b []byte) (v []byte, n int)

// EncodeBool encodes a bool as a uint64
func EncodeBool(x bool) uint64

// DecodeBool decodes a uint64 as a bool
func DecodeBool(x uint64) bool

// EncodeZigZag encodes an int64 as a zig-zag-encoded uint64
func EncodeZigZag(x int64) uint64

// DecodeZigZag decodes a zig-zag-encoded uint64 as an int64
func DecodeZigZag(x uint64) int64

// EncodeTag encodes the field Number and wire Type into its unified form
func EncodeTag(num Number, typ Type) uint64

// DecodeTag decodes the field Number and wire Type from its unified form
// The Number is -1 if the decoded field number overflows int32
// Other than overflow, this does not check for field number validity
func DecodeTag(x uint64) (Number, Type)

// SizeVarint returns the encoded size of a varint
// The size is guaranteed to be within 1 and 10, inclusive
func SizeVarint(v uint64) int

// SizeFixed32 returns the encoded size of a fixed32; which is always 4
func SizeFixed32() int

// SizeFixed64 returns the encoded size of a fixed64; which is always 8
func SizeFixed64() int

// SizeBytes returns the encoded size of a length-prefixed bytes value,
// given only the length
func SizeBytes(n int) int

// SizeTag returns the encoded size of a field tag
func SizeTag(num Number) int

// SizeGroup returns the encoded size of a group, given only the length
func SizeGroup(num Number, n int) int

// ParseError converts an error code into an error value
// This returns nil if n is a non-negative number
func ParseError(n int) error

// MarshalTo writes a varint size-delimited wire-format message to w with the
// default options
func MarshalTo(w io.Writer, m proto.Message) (int, error)

import (
    "bytes"
    "google.golang.org/protobuf/encoding/protodelim"
)

var buf bytes.Buffer

// Write multiple messages to the same stream
for _, msg := range messages {
    _, err := protodelim.MarshalTo(&buf, msg)
    if err != nil {
        log.Fatal(err)
    }
}

// MarshalOptions is a configurable varint size-delimited marshaler
type MarshalOptions struct{ proto.MarshalOptions }

// MarshalTo writes a varint size-delimited wire-format message to w
// If w returns an error, MarshalTo returns it unchanged
func (o MarshalOptions) MarshalTo(w io.Writer, m proto.Message) (int, error)

// Reader is the interface expected by UnmarshalFrom
// It is implemented by *bufio.Reader
type Reader interface {
    io.Reader
    io.ByteReader
}

// UnmarshalFrom parses and consumes a varint size-delimited wire-format
// message from r with the default options
// The provided message must be mutable (e.g., a non-nil pointer to a message)
func UnmarshalFrom(r Reader, m proto.Message) error

import (
    "bufio"
    "google.golang.org/protobuf/encoding/protodelim"
)

reader := bufio.NewReader(file)

// Read multiple messages from the same stream
for {
    msg := &pb.MyMessage{}
    err := protodelim.UnmarshalFrom(reader, msg)
    if err == io.EOF {
        break
    }
    if err != nil {
        log.Fatal(err)
    }
    // Process msg
}

// UnmarshalOptions is a configurable varint size-delimited unmarshaler
type UnmarshalOptions struct {
    proto.UnmarshalOptions

    // MaxSize is the maximum size in wire-format bytes of a single message.
    // Unmarshaling a message larger than MaxSize will return an error.
    // A zero MaxSize will default to 4 MiB.
    // Setting MaxSize to -1 disables the limit.
    MaxSize int64
}

// UnmarshalFrom parses and consumes a varint size-delimited wire-format
// message from r. The provided message must be mutable (e.g., a non-nil
// pointer to a message).
//
// The error is io.EOF error only if no bytes are read. If an EOF happens
// after reading some but not all the bytes, UnmarshalFrom returns a non-io.EOF
// error. In particular if r returns a non-io.EOF error, UnmarshalFrom returns
// it unchanged, and if only a size is read with no subsequent message,
// io.ErrUnexpectedEOF is returned.
func (o UnmarshalOptions) UnmarshalFrom(r Reader, m proto.Message) error

// SizeTooLargeError is an error that is returned when the unmarshaler
// encounters a message size that is larger than its configured MaxSize
type SizeTooLargeError struct {
    // Size is the varint size of the message encountered
    // that was larger than the provided MaxSize.
    Size uint64

    // MaxSize is the MaxSize limit configured in UnmarshalOptions, which Size exceeded.
    MaxSize uint64
}

func (e *SizeTooLargeError) Error() string

opts := protodelim.UnmarshalOptions{
    MaxSize: 1024 * 1024, // 1 MiB limit
}

msg := &pb.MyMessage{}
err := opts.UnmarshalFrom(reader, msg)
if err != nil {
    var sizeErr *protodelim.SizeTooLargeError
    if errors.As(err, &sizeErr) {
        fmt.Printf("Message too large: %d bytes (limit: %d)\n",
            sizeErr.Size, sizeErr.MaxSize)
    }
}