marshal.go

package main

import (
    "log"
    "io/ioutil"


    "github.com/golang/protobuf/proto"
    "./example"
)

func main() {
    n := (1 << 29) + 4096

    test := &example.Test {
        Data: make([]int64, n),
    }

    for i := 0; i < n; i++ {
        test.Data[i] = int64(i)
    }
    data, err := proto.Marshal(test)
    if err != nil {
        log.Fatal("marshaling error: ", err)
    }
    ioutil.WriteFile("large.pb", data, 0644)
}

memory_usage

$ mkdir example
$ mv test* example
$ /usr/bin/time -v go run marshal.go
	Command being timed: "go run marshal.go"
	User time (seconds): 28.44
	System time (seconds): 9.97
	Percent of CPU this job got: 51%
	Elapsed (wall clock) time (h:mm:ss or m:ss): 1:14.78
......
	Maximum resident set size (kbytes): 13177628 # (13.1776 G)
......	
	Exit status: 0
$ /usr/bin/time -v go run unmarshal.go
	Command being timed: "go run unmarshal.go"
	User time (seconds): 28.09
	System time (seconds): 25.28
	Percent of CPU this job got: 89%
	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:59.73
......
	Maximum resident set size (kbytes): 20805872 # (20.8059 G)
......
	Exit status: 0
$ ls -lh large.pb
-rw-r--r-- 1 zifei zifei 2.8G  6月 18 12:58 large.pb

run.sh

#!/bin/bash

protoc --gogo_out=. -I$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. test.proto
mkdir example
mv test* example
/usr/bin/time -v go run marshal.go
ls -lh large.pb
/usr/bin/time -v go run unmarshal.go

test.pb.go

// Code generated by protoc-gen-gogo.
// source: test.proto
// DO NOT EDIT!

/*
Package example is a generated protocol buffer package.

It is generated from these files:
	test.proto

It has these top-level messages:
	Test
*/
package example

import proto "github.com/gogo/protobuf/proto"

// discarding unused import gogoproto "github.com/gogo/protobuf/gogoproto/gogo.pb"

import io "io"
import fmt "fmt"

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal

type Test struct {
	Data []int64 `protobuf:"varint,1,rep,name=data" json:"data,omitempty"`
}

func (m *Test) Reset()         { *m = Test{} }
func (m *Test) String() string { return proto.CompactTextString(m) }
func (*Test) ProtoMessage()    {}

func init() {
}
func (m *Test) Unmarshal(data []byte) error {
	l := len(data)
	iNdEx := 0
	for iNdEx < l {
		var wire uint64
		for shift := uint(0); ; shift += 7 {
			if iNdEx >= l {
				return io.ErrUnexpectedEOF
			}
			b := data[iNdEx]
			iNdEx++
			wire |= (uint64(b) & 0x7F) << shift
			if b < 0x80 {
				break
			}
		}
		fieldNum := int32(wire >> 3)
		wireType := int(wire & 0x7)
		switch fieldNum {
		case 1:
			if wireType != 0 {
				return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
			}
			var v int64
			for shift := uint(0); ; shift += 7 {
				if iNdEx >= l {
					return io.ErrUnexpectedEOF
				}
				b := data[iNdEx]
				iNdEx++
				v |= (int64(b) & 0x7F) << shift
				if b < 0x80 {
					break
				}
			}
			m.Data = append(m.Data, v)
		default:
			var sizeOfWire int
			for {
				sizeOfWire++
				wire >>= 7
				if wire == 0 {
					break
				}
			}
			iNdEx -= sizeOfWire
			skippy, err := skipTest(data[iNdEx:])
			if err != nil {
				return err
			}
			if (iNdEx + skippy) > l {
				return io.ErrUnexpectedEOF
			}
			iNdEx += skippy
		}
	}

	return nil
}
func skipTest(data []byte) (n int, err error) {
	l := len(data)
	iNdEx := 0
	for iNdEx < l {
		var wire uint64
		for shift := uint(0); ; shift += 7 {
			if iNdEx >= l {
				return 0, io.ErrUnexpectedEOF
			}
			b := data[iNdEx]
			iNdEx++
			wire |= (uint64(b) & 0x7F) << shift
			if b < 0x80 {
				break
			}
		}
		wireType := int(wire & 0x7)
		switch wireType {
		case 0:
			for {
				if iNdEx >= l {
					return 0, io.ErrUnexpectedEOF
				}
				iNdEx++
				if data[iNdEx-1] < 0x80 {
					break
				}
			}
			return iNdEx, nil
		case 1:
			iNdEx += 8
			return iNdEx, nil
		case 2:
			var length int
			for shift := uint(0); ; shift += 7 {
				if iNdEx >= l {
					return 0, io.ErrUnexpectedEOF
				}
				b := data[iNdEx]
				iNdEx++
				length |= (int(b) & 0x7F) << shift
				if b < 0x80 {
					break
				}
			}
			iNdEx += length
			return iNdEx, nil
		case 3:
			for {
				var wire uint64
				var start int = iNdEx
				for shift := uint(0); ; shift += 7 {
					if iNdEx >= l {
						return 0, io.ErrUnexpectedEOF
					}
					b := data[iNdEx]
					iNdEx++
					wire |= (uint64(b) & 0x7F) << shift
					if b < 0x80 {
						break
					}
				}
				wireType := int(wire & 0x7)
				if wireType == 4 {
					break
				}
				next, err := skipTest(data[start:])
				if err != nil {
					return 0, err
				}
				iNdEx = start + next
			}
			return iNdEx, nil
		case 4:
			return iNdEx, nil
		case 5:
			iNdEx += 4
			return iNdEx, nil
		default:
			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
		}
	}
	panic("unreachable")
}
func (m *Test) Size() (n int) {
	var l int
	_ = l
	if len(m.Data) > 0 {
		for _, e := range m.Data {
			n += 1 + sovTest(uint64(e))
		}
	}
	return n
}

func sovTest(x uint64) (n int) {
	for {
		n++
		x >>= 7
		if x == 0 {
			break
		}
	}
	return n
}
func sozTest(x uint64) (n int) {
	return sovTest(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Test) Marshal() (data []byte, err error) {
	size := m.Size()
	data = make([]byte, size)
	n, err := m.MarshalTo(data)
	if err != nil {
		return nil, err
	}
	return data[:n], nil
}

func (m *Test) MarshalTo(data []byte) (n int, err error) {
	var i int
	_ = i
	var l int
	_ = l
	if len(m.Data) > 0 {
		for _, num := range m.Data {
			data[i] = 0x8
			i++
			i = encodeVarintTest(data, i, uint64(num))
		}
	}
	return i, nil
}

func encodeFixed64Test(data []byte, offset int, v uint64) int {
	data[offset] = uint8(v)
	data[offset+1] = uint8(v >> 8)
	data[offset+2] = uint8(v >> 16)
	data[offset+3] = uint8(v >> 24)
	data[offset+4] = uint8(v >> 32)
	data[offset+5] = uint8(v >> 40)
	data[offset+6] = uint8(v >> 48)
	data[offset+7] = uint8(v >> 56)
	return offset + 8
}
func encodeFixed32Test(data []byte, offset int, v uint32) int {
	data[offset] = uint8(v)
	data[offset+1] = uint8(v >> 8)
	data[offset+2] = uint8(v >> 16)
	data[offset+3] = uint8(v >> 24)
	return offset + 4
}
func encodeVarintTest(data []byte, offset int, v uint64) int {
	for v >= 1<<7 {
		data[offset] = uint8(v&0x7f | 0x80)
		v >>= 7
		offset++
	}
	data[offset] = uint8(v)
	return offset + 1
}

test.proto

syntax = "proto3";

package example;

import "github.com/gogo/protobuf/gogoproto/gogo.proto";

option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.goproto_getters_all) = false;

message Test {
  repeated int64 data = 1;
}

unmarshal.go

package main

import (
    "log"
    "io/ioutil"


    "github.com/gogo/protobuf/proto"
    "./example"
)

func main() {
     n := (1 << 29) + 4096
     data, err := ioutil.ReadFile("large.pb")
     newTest := &example.Test{}
     err = proto.Unmarshal(data, newTest)
     if err != nil {
         log.Fatal("unmarshaling error: ", err)
     }

     for i := 0; i < n; i++ {
         if newTest.Data[i] != int64(i) {
             log.Fatalf("data mismatch %q != %q", newTest.Data[i], true)
         }
     }
}