nune/gitea
nune/gitea
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[Vendor] update macaron related (#13409)

Browse source 
* Vendor: update gitea.com/macaron/session to a177a270

* make vendor

* Vendor: update gitea.com/macaron/macaron to 0db5d458

* make vendor

* Vendor: update gitea.com/macaron/cache to 905232fb

* make vendor

* Vendor: update gitea.com/macaron/i18n to 4ca3dd0c

* make vendor

* Vendor: update gitea.com/macaron/gzip to efa5e847

* make vendor

* Vendor: update gitea.com/macaron/captcha to e8597820

* make vendor
This commit is contained in:
6543 2020-11-03 07:04:09 +01:00 • committed by GitHub
parent b687707014
commit 70ea2300ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
118 changed files with 14557 additions and 6115 deletions
Download patch file Download diff file Expand all files Collapse all files

8
vendor/github.com/couchbase/gomemcached/client/collections_filter.go generated vendored
View file

@ -45,7 +45,7 @@ type streamIdNonResumeScopeMeta struct {
}
func (c *CollectionsFilter) IsValid() error {
if c.UseManifestUid {
if c.UseManifestUid && c.UseStreamId {
return fmt.Errorf("Not implemented yet")
}
@ -99,8 +99,10 @@ func (c *CollectionsFilter) ToStreamReqBody() ([]byte, error) {
case false:
switch c.UseManifestUid {
case true:
// TODO
return nil, fmt.Errorf("NotImplemented1")
filter := &nonStreamIdResumeScopeMeta{
ManifestId: fmt.Sprintf("%x", c.ManifestUid),
}
output = *filter
case false:
switch len(c.CollectionsList) > 0 {
case true:

415
vendor/github.com/couchbase/gomemcached/client/mc.go generated vendored
View file

@ -19,8 +19,8 @@ import (
)
type ClientIface interface {
Add(vb uint16, key string, flags int, exp int, body []byte) (*gomemcached.MCResponse, error)
Append(vb uint16, key string, data []byte) (*gomemcached.MCResponse, error)
Add(vb uint16, key string, flags int, exp int, body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error)
Append(vb uint16, key string, data []byte, context ...*ClientContext) (*gomemcached.MCResponse, error)
Auth(user, pass string) (*gomemcached.MCResponse, error)
AuthList() (*gomemcached.MCResponse, error)
AuthPlain(user, pass string) (*gomemcached.MCResponse, error)
@ -30,44 +30,87 @@ type ClientIface interface {
CollectionsGetCID(scope string, collection string) (*gomemcached.MCResponse, error)
CollectionEnabled() bool
Close() error
Decr(vb uint16, key string, amt, def uint64, exp int) (uint64, error)
Del(vb uint16, key string) (*gomemcached.MCResponse, error)
Decr(vb uint16, key string, amt, def uint64, exp int, context ...*ClientContext) (uint64, error)
Del(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error)
EnableMutationToken() (*gomemcached.MCResponse, error)
EnableFeatures(features Features) (*gomemcached.MCResponse, error)
Get(vb uint16, key string) (*gomemcached.MCResponse, error)
Get(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error)
GetAllVbSeqnos(vbSeqnoMap map[uint16]uint64, context ...*ClientContext) (map[uint16]uint64, error)
GetAndTouch(vb uint16, key string, exp int, context ...*ClientContext) (*gomemcached.MCResponse, error)
GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MCResponse, subPaths []string, context ...*ClientContext) error
GetCollectionsManifest() (*gomemcached.MCResponse, error)
GetFromCollection(vb uint16, cid uint32, key string) (*gomemcached.MCResponse, error)
GetSubdoc(vb uint16, key string, subPaths []string) (*gomemcached.MCResponse, error)
GetAndTouch(vb uint16, key string, exp int) (*gomemcached.MCResponse, error)
GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MCResponse, subPaths []string) error
GetMeta(vb uint16, key string) (*gomemcached.MCResponse, error)
GetRandomDoc() (*gomemcached.MCResponse, error)
GetMeta(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error)
GetRandomDoc(context ...*ClientContext) (*gomemcached.MCResponse, error)
GetSubdoc(vb uint16, key string, subPaths []string, context ...*ClientContext) (*gomemcached.MCResponse, error)
Hijack() io.ReadWriteCloser
Incr(vb uint16, key string, amt, def uint64, exp int) (uint64, error)
Incr(vb uint16, key string, amt, def uint64, exp int, context ...*ClientContext) (uint64, error)
Observe(vb uint16, key string) (result ObserveResult, err error)
ObserveSeq(vb uint16, vbuuid uint64) (result *ObserveSeqResult, err error)
Receive() (*gomemcached.MCResponse, error)
ReceiveWithDeadline(deadline time.Time) (*gomemcached.MCResponse, error)
Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error)
Set(vb uint16, key string, flags int, exp int, body []byte) (*gomemcached.MCResponse, error)
Set(vb uint16, key string, flags int, exp int, body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error)
SetKeepAliveOptions(interval time.Duration)
SetReadDeadline(t time.Time)
SetDeadline(t time.Time)
SelectBucket(bucket string) (*gomemcached.MCResponse, error)
SetCas(vb uint16, key string, flags int, exp int, cas uint64, body []byte) (*gomemcached.MCResponse, error)
SetCas(vb uint16, key string, flags int, exp int, cas uint64, body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error)
Stats(key string) ([]StatValue, error)
StatsMap(key string) (map[string]string, error)
StatsMapForSpecifiedStats(key string, statsMap map[string]string) error
Transmit(req *gomemcached.MCRequest) error
TransmitWithDeadline(req *gomemcached.MCRequest, deadline time.Time) error
TransmitResponse(res *gomemcached.MCResponse) error
UprGetFailoverLog(vb []uint16) (map[uint16]*FailoverLog, error)
// UprFeed Related
NewUprFeed() (*UprFeed, error)
NewUprFeedIface() (UprFeedIface, error)
NewUprFeedWithConfig(ackByClient bool) (*UprFeed, error)
NewUprFeedWithConfigIface(ackByClient bool) (UprFeedIface, error)
UprGetFailoverLog(vb []uint16) (map[uint16]*FailoverLog, error)
}
type ClientContext struct {
// Collection-based context
CollId uint32
// VB-state related context
// nil means not used in this context
VbState *VbStateType
}
type VbStateType uint8
const (
VbAlive VbStateType = 0x00
VbActive VbStateType = 0x01
VbReplica VbStateType = 0x02
VbPending VbStateType = 0x03
VbDead VbStateType = 0x04
)
func (context *ClientContext) InitExtras(req *gomemcached.MCRequest, client *Client) {
if req == nil || client == nil {
return
}
var bytesToAllocate int
switch req.Opcode {
case gomemcached.GET_ALL_VB_SEQNOS:
if context.VbState != nil {
bytesToAllocate += 4
}
if client.CollectionEnabled() {
if context.VbState == nil {
bytesToAllocate += 8
} else {
bytesToAllocate += 4
}
}
}
if bytesToAllocate > 0 {
req.Extras = make([]byte, bytesToAllocate)
}
}
const bufsize = 1024
@ -102,8 +145,8 @@ type Client struct {
hdrBuf []byte
featureMtx sync.RWMutex
sentHeloFeatures Features
collectionsEnabled uint32
deadline time.Time
}
var (
@ -156,7 +199,11 @@ func (c *Client) SetReadDeadline(t time.Time) {
}
func (c *Client) SetDeadline(t time.Time) {
if t.Equal(c.deadline) {
return
}
c.conn.SetDeadline(t)
c.deadline = t
}
// Wrap an existing transport.
@ -287,60 +334,103 @@ func (c *Client) EnableMutationToken() (*gomemcached.MCResponse, error) {
//Send a hello command to enable specific features
func (c *Client) EnableFeatures(features Features) (*gomemcached.MCResponse, error) {
var payload []byte
collectionsEnabled := 0
for _, feature := range features {
if feature == FeatureCollections {
collectionsEnabled = 1
}
payload = append(payload, 0, 0)
binary.BigEndian.PutUint16(payload[len(payload)-2:], uint16(feature))
}
c.featureMtx.Lock()
c.sentHeloFeatures = features
c.featureMtx.Unlock()
return c.Send(&gomemcached.MCRequest{
rv, err := c.Send(&gomemcached.MCRequest{
Opcode: gomemcached.HELLO,
Key: []byte("GoMemcached"),
Body: payload,
})
if err == nil && collectionsEnabled != 0 {
atomic.StoreUint32(&c.collectionsEnabled, uint32(collectionsEnabled))
}
return rv, err
}
// Sets collection info for a request
func (c *Client) setCollection(req *gomemcached.MCRequest, context ...*ClientContext) error {
req.CollIdLen = 0
collectionId := uint32(0)
if len(context) > 0 {
collectionId = context[0].CollId
}
// if the optional collection is specified, it must be default for clients that haven't turned on collections
if atomic.LoadUint32(&c.collectionsEnabled) == 0 {
if collectionId != 0 {
return fmt.Errorf("Client does not use collections but a collection was specified")
}
} else {
req.CollIdLen = binary.PutUvarint(req.CollId[:], uint64(collectionId))
}
return nil
}
func (c *Client) setVbSeqnoContext(req *gomemcached.MCRequest, context ...*ClientContext) error {
if len(context) == 0 || req == nil {
return nil
}
switch req.Opcode {
case gomemcached.GET_ALL_VB_SEQNOS:
if len(context) == 0 {
return nil
}
if len(req.Extras) == 0 {
context[0].InitExtras(req, c)
}
if context[0].VbState != nil {
binary.BigEndian.PutUint32(req.Extras, uint32(*(context[0].VbState)))
}
if c.CollectionEnabled() {
binary.BigEndian.PutUint32(req.Extras[4:8], context[0].CollId)
}
return nil
default:
return fmt.Errorf("setVbState Not supported for opcode: %v", req.Opcode.String())
}
}
// Get the value for a key.
func (c *Client) Get(vb uint16, key string) (*gomemcached.MCResponse, error) {
return c.Send(&gomemcached.MCRequest{
func (c *Client) Get(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.GET,
VBucket: vb,
Key: []byte(key),
})
}
// Get the value for a key from a collection, identified by collection id.
func (c *Client) GetFromCollection(vb uint16, cid uint32, key string) (*gomemcached.MCResponse, error) {
keyBytes := []byte(key)
encodedCid := make([]byte, binary.MaxVarintLen32)
lenEncodedCid := binary.PutUvarint(encodedCid, uint64(cid))
encodedKey := make([]byte, 0, lenEncodedCid+len(keyBytes))
encodedKey = append(encodedKey, encodedCid[0:lenEncodedCid]...)
encodedKey = append(encodedKey, keyBytes...)
return c.Send(&gomemcached.MCRequest{
Opcode: gomemcached.GET,
VBucket: vb,
Key: encodedKey,
})
}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
return c.Send(req)
}
// Get the xattrs, doc value for the input key
func (c *Client) GetSubdoc(vb uint16, key string, subPaths []string) (*gomemcached.MCResponse, error) {
func (c *Client) GetSubdoc(vb uint16, key string, subPaths []string, context ...*ClientContext) (*gomemcached.MCResponse, error) {
extraBuf, valueBuf := GetSubDocVal(subPaths)
res, err := c.Send(&gomemcached.MCRequest{
req := &gomemcached.MCRequest{
Opcode: gomemcached.SUBDOC_MULTI_LOOKUP,
VBucket: vb,
Key: []byte(key),
Extras: extraBuf,
Body: valueBuf,
})
}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
res, err := c.Send(req)
if err != nil && IfResStatusError(res) {
return res, err
@ -376,48 +466,56 @@ func (c *Client) CollectionsGetCID(scope string, collection string) (*gomemcache
}
func (c *Client) CollectionEnabled() bool {
c.featureMtx.RLock()
defer c.featureMtx.RUnlock()
for _, feature := range c.sentHeloFeatures {
if feature == FeatureCollections {
return true
}
}
return false
return atomic.LoadUint32(&c.collectionsEnabled) > 0
}
// Get the value for a key, and update expiry
func (c *Client) GetAndTouch(vb uint16, key string, exp int) (*gomemcached.MCResponse, error) {
func (c *Client) GetAndTouch(vb uint16, key string, exp int, context ...*ClientContext) (*gomemcached.MCResponse, error) {
extraBuf := make([]byte, 4)
binary.BigEndian.PutUint32(extraBuf[0:], uint32(exp))
return c.Send(&gomemcached.MCRequest{
req := &gomemcached.MCRequest{
Opcode: gomemcached.GAT,
VBucket: vb,
Key: []byte(key),
Extras: extraBuf,
})
}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
return c.Send(req)
}
// Get metadata for a key
func (c *Client) GetMeta(vb uint16, key string) (*gomemcached.MCResponse, error) {
return c.Send(&gomemcached.MCRequest{
func (c *Client) GetMeta(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.GET_META,
VBucket: vb,
Key: []byte(key),
})
}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
return c.Send(req)
}
// Del deletes a key.
func (c *Client) Del(vb uint16, key string) (*gomemcached.MCResponse, error) {
return c.Send(&gomemcached.MCRequest{
func (c *Client) Del(vb uint16, key string, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.DELETE,
VBucket: vb,
Key: []byte(key)})
Key: []byte(key),
}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
return c.Send(req)
}
// Get a random document
func (c *Client) GetRandomDoc() (*gomemcached.MCResponse, error) {
func (c *Client) GetRandomDoc(context ...*ClientContext) (*gomemcached.MCResponse, error) {
return c.Send(&gomemcached.MCRequest{
Opcode: 0xB6,
})
@ -522,8 +620,7 @@ func (c *Client) SelectBucket(bucket string) (*gomemcached.MCResponse, error) {
}
func (c *Client) store(opcode gomemcached.CommandCode, vb uint16,
key string, flags int, exp int, body []byte) (*gomemcached.MCResponse, error) {
key string, flags int, exp int, body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: opcode,
VBucket: vb,
@ -533,13 +630,16 @@ func (c *Client) store(opcode gomemcached.CommandCode, vb uint16,
Extras: []byte{0, 0, 0, 0, 0, 0, 0, 0},
Body: body}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
return c.Send(req)
}
func (c *Client) storeCas(opcode gomemcached.CommandCode, vb uint16,
key string, flags int, exp int, cas uint64, body []byte) (*gomemcached.MCResponse, error) {
key string, flags int, exp int, cas uint64, body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: opcode,
VBucket: vb,
@ -549,20 +649,29 @@ func (c *Client) storeCas(opcode gomemcached.CommandCode, vb uint16,
Extras: []byte{0, 0, 0, 0, 0, 0, 0, 0},
Body: body}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
return c.Send(req)
}
// Incr increments the value at the given key.
func (c *Client) Incr(vb uint16, key string,
amt, def uint64, exp int) (uint64, error) {
amt, def uint64, exp int, context ...*ClientContext) (uint64, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.INCREMENT,
VBucket: vb,
Key: []byte(key),
Extras: make([]byte, 8+8+4),
}
err := c.setCollection(req, context...)
if err != nil {
return 0, err
}
binary.BigEndian.PutUint64(req.Extras[:8], amt)
binary.BigEndian.PutUint64(req.Extras[8:16], def)
binary.BigEndian.PutUint32(req.Extras[16:20], uint32(exp))
@ -577,14 +686,18 @@ func (c *Client) Incr(vb uint16, key string,
// Decr decrements the value at the given key.
func (c *Client) Decr(vb uint16, key string,
amt, def uint64, exp int) (uint64, error) {
amt, def uint64, exp int, context ...*ClientContext) (uint64, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.DECREMENT,
VBucket: vb,
Key: []byte(key),
Extras: make([]byte, 8+8+4),
}
err := c.setCollection(req, context...)
if err != nil {
return 0, err
}
binary.BigEndian.PutUint64(req.Extras[:8], amt)
binary.BigEndian.PutUint64(req.Extras[8:16], def)
binary.BigEndian.PutUint32(req.Extras[16:20], uint32(exp))
@ -599,24 +712,24 @@ func (c *Client) Decr(vb uint16, key string,
// Add a value for a key (store if not exists).
func (c *Client) Add(vb uint16, key string, flags int, exp int,
body []byte) (*gomemcached.MCResponse, error) {
return c.store(gomemcached.ADD, vb, key, flags, exp, body)
body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
return c.store(gomemcached.ADD, vb, key, flags, exp, body, context...)
}
// Set the value for a key.
func (c *Client) Set(vb uint16, key string, flags int, exp int,
body []byte) (*gomemcached.MCResponse, error) {
return c.store(gomemcached.SET, vb, key, flags, exp, body)
body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
return c.store(gomemcached.SET, vb, key, flags, exp, body, context...)
}
// SetCas set the value for a key with cas
func (c *Client) SetCas(vb uint16, key string, flags int, exp int, cas uint64,
body []byte) (*gomemcached.MCResponse, error) {
return c.storeCas(gomemcached.SET, vb, key, flags, exp, cas, body)
body []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
return c.storeCas(gomemcached.SET, vb, key, flags, exp, cas, body, context...)
}
// Append data to the value of a key.
func (c *Client) Append(vb uint16, key string, data []byte) (*gomemcached.MCResponse, error) {
func (c *Client) Append(vb uint16, key string, data []byte, context ...*ClientContext) (*gomemcached.MCResponse, error) {
req := &gomemcached.MCRequest{
Opcode: gomemcached.APPEND,
VBucket: vb,
@ -625,11 +738,15 @@ func (c *Client) Append(vb uint16, key string, data []byte) (*gomemcached.MCResp
Opaque: 0,
Body: data}
err := c.setCollection(req, context...)
if err != nil {
return nil, err
}
return c.Send(req)
}
// GetBulk gets keys in bulk
func (c *Client) GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MCResponse, subPaths []string) error {
func (c *Client) GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MCResponse, subPaths []string, context ...*ClientContext) error {
stopch := make(chan bool)
var wg sync.WaitGroup
@ -698,6 +815,10 @@ func (c *Client) GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MC
Opcode: gomemcached.GET,
VBucket: vb,
}
err := c.setCollection(memcachedReqPkt, context...)
if err != nil {
return err
}
if len(subPaths) > 0 {
extraBuf, valueBuf := GetSubDocVal(subPaths)
@ -719,7 +840,7 @@ func (c *Client) GetBulk(vb uint16, keys []string, rv map[string]*gomemcached.MC
} // End of Get request
// finally transmit a NOOP
err := c.Transmit(&gomemcached.MCRequest{
err = c.Transmit(&gomemcached.MCRequest{
Opcode: gomemcached.NOOP,
VBucket: vb,
Opaque: c.opaque,
@ -747,7 +868,10 @@ func GetSubDocVal(subPaths []string) (extraBuf, valueBuf []byte) {
}
// Xattr retrieval - subdoc multi get
extraBuf = append(extraBuf, uint8(0x04))
// Set deleted true only if it is not expiration
if len(subPaths) != 1 || subPaths[0] != "$document.exptime" {
extraBuf = append(extraBuf, uint8(0x04))
}
valueBuf = make([]byte, num*4+totalBytesLen)
@ -1138,6 +1262,38 @@ func (c *Client) StatsMapForSpecifiedStats(key string, statsMap map[string]strin
return nil
}
// UprGetFailoverLog for given list of vbuckets.
func (mc *Client) UprGetFailoverLog(vb []uint16) (map[uint16]*FailoverLog, error) {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_FAILOVERLOG,
Opaque: opaqueFailover,
}
failoverLogs := make(map[uint16]*FailoverLog)
for _, vBucket := range vb {
rq.VBucket = vBucket
if err := mc.Transmit(rq); err != nil {
return nil, err
}
res, err := mc.Receive()
if err != nil {
return nil, fmt.Errorf("failed to receive %s", err.Error())
} else if res.Opcode != gomemcached.UPR_FAILOVERLOG || res.Status != gomemcached.SUCCESS {
return nil, fmt.Errorf("unexpected #opcode %v", res.Opcode)
}
flog, err := parseFailoverLog(res.Body)
if err != nil {
return nil, fmt.Errorf("unable to parse failover logs for vb %d", vb)
}
failoverLogs[vBucket] = flog
}
return failoverLogs, nil
}
// Hijack exposes the underlying connection from this client.
//
// It also marks the connection as unhealthy since the client will
@ -1166,3 +1322,98 @@ func IfResStatusError(response *gomemcached.MCResponse) bool {
func (c *Client) Conn() io.ReadWriteCloser {
return c.conn
}
// Since the binary request supports only a single collection at a time, it is possible
// that this may be called multiple times in succession by callers to get vbSeqnos for
// multiple collections. Thus, caller could pass in a non-nil map so the gomemcached
// client won't need to allocate new map for each call to prevent too much GC
// NOTE: If collection is enabled and context is not given, KV will still return stats for default collection
func (c *Client) GetAllVbSeqnos(vbSeqnoMap map[uint16]uint64, context ...*ClientContext) (map[uint16]uint64, error) {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.GET_ALL_VB_SEQNOS,
Opaque: opaqueGetSeqno,
}
err := c.setVbSeqnoContext(rq, context...)
if err != nil {
return vbSeqnoMap, err
}
err = c.Transmit(rq)
if err != nil {
return vbSeqnoMap, err
}
res, err := c.Receive()
if err != nil {
return vbSeqnoMap, fmt.Errorf("failed to receive: %v", err)
}
vbSeqnosList, err := parseGetSeqnoResp(res.Body)
if err != nil {
logging.Errorf("Unable to parse : err: %v\n", err)
return vbSeqnoMap, err
}
if vbSeqnoMap == nil {
vbSeqnoMap = make(map[uint16]uint64)
}
combineMapWithReturnedList(vbSeqnoMap, vbSeqnosList)
return vbSeqnoMap, nil
}
func combineMapWithReturnedList(vbSeqnoMap map[uint16]uint64, list *VBSeqnos) {
if list == nil {
return
}
// If the map contains exactly the existing vbs in the list, no need to modify
needToCleanupMap := true
if len(vbSeqnoMap) == 0 {
needToCleanupMap = false
} else if len(vbSeqnoMap) == len(*list) {
needToCleanupMap = false
for _, pair := range *list {
_, vbExists := vbSeqnoMap[uint16(pair[0])]
if !vbExists {
needToCleanupMap = true
break
}
}
}
if needToCleanupMap {
var vbsToDelete []uint16
for vbInSeqnoMap, _ := range vbSeqnoMap {
// If a vb in the seqno map doesn't exist in the returned list, need to clean up
// to ensure returning an accurate result
found := false
var vbno uint16
for _, pair := range *list {
vbno = uint16(pair[0])
if vbno == vbInSeqnoMap {
found = true
break
} else if vbno > vbInSeqnoMap {
// definitely not in the list
break
}
}
if !found {
vbsToDelete = append(vbsToDelete, vbInSeqnoMap)
}
}
for _, vbno := range vbsToDelete {
delete(vbSeqnoMap, vbno)
}
}
// Set the map with data from the list
for _, pair := range *list {
vbno := uint16(pair[0])
seqno := pair[1]
vbSeqnoMap[vbno] = seqno
}
}

19
vendor/github.com/couchbase/gomemcached/client/upr_event.go generated vendored
View file

@ -89,6 +89,9 @@ type UprEvent struct {
// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64
// Containing a pair of vbno and the high seqno
type VBSeqnos [][2]uint64
func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
event := &UprEvent{
Opcode: rq.Opcode,
@ -148,6 +151,8 @@ func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFrom
event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
} else if event.IsSystemEvent() {
event.PopulateEvent(rq.Extras)
} else if event.IsSeqnoAdv() {
event.PopulateSeqnoAdv(rq.Extras)
}
return event
@ -199,17 +204,31 @@ func (event *UprEvent) IsSystemEvent() bool {
return event.Opcode == gomemcached.DCP_SYSTEM_EVENT
}
func (event *UprEvent) IsSeqnoAdv() bool {
return event.Opcode == gomemcached.DCP_SEQNO_ADV
}
func (event *UprEvent) PopulateEvent(extras []byte) {
if len(extras) < dcpSystemEventExtraLen {
// Wrong length, don't parse
return
}
event.Seqno = binary.BigEndian.Uint64(extras[:8])
event.SystemEvent = SystemEventType(binary.BigEndian.Uint32(extras[8:12]))
var versionTemp uint16 = binary.BigEndian.Uint16(extras[12:14])
event.SysEventVersion = uint8(versionTemp >> 8)
}
func (event *UprEvent) PopulateSeqnoAdv(extras []byte) {
if len(extras) < dcpSeqnoAdvExtraLen {
// Wrong length, don't parse
return
}
event.Seqno = binary.BigEndian.Uint64(extras[:8])
}
func (event *UprEvent) GetSystemEventName() (string, error) {
switch event.SystemEvent {
case CollectionCreate:

67
vendor/github.com/couchbase/gomemcached/client/upr_feed.go generated vendored
View file

@ -20,9 +20,11 @@ const uprDeletetionExtraLen = 18
const uprDeletetionWithDeletionTimeExtraLen = 21
const uprSnapshotExtraLen = 20
const dcpSystemEventExtraLen = 13
const dcpSeqnoAdvExtraLen = 8
const bufferAckThreshold = 0.2
const opaqueOpen = 0xBEAF0001
const opaqueFailover = 0xDEADBEEF
const opaqueGetSeqno = 0xDEADBEEF
const uprDefaultNoopInterval = 120
// Counter on top of opaqueOpen that others can draw from for open and control msgs
@ -605,44 +607,6 @@ func (feed *UprFeed) uprOpen(name string, sequence uint32, bufSize uint32, featu
return
}
// UprGetFailoverLog for given list of vbuckets.
func (mc *Client) UprGetFailoverLog(
vb []uint16) (map[uint16]*FailoverLog, error) {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_FAILOVERLOG,
Opaque: opaqueFailover,
}
var allFeaturesDisabled UprFeatures
if err := doUprOpen(mc, "FailoverLog", 0, allFeaturesDisabled); err != nil {
return nil, fmt.Errorf("UPR_OPEN Failed %s", err.Error())
}
failoverLogs := make(map[uint16]*FailoverLog)
for _, vBucket := range vb {
rq.VBucket = vBucket
if err := mc.Transmit(rq); err != nil {
return nil, err
}
res, err := mc.Receive()
if err != nil {
return nil, fmt.Errorf("failed to receive %s", err.Error())
} else if res.Opcode != gomemcached.UPR_FAILOVERLOG || res.Status != gomemcached.SUCCESS {
return nil, fmt.Errorf("unexpected #opcode %v", res.Opcode)
}
flog, err := parseFailoverLog(res.Body)
if err != nil {
return nil, fmt.Errorf("unable to parse failover logs for vb %d", vb)
}
failoverLogs[vBucket] = flog
}
return failoverLogs, nil
}
// UprRequestStream for a single vbucket.
func (feed *UprFeed) UprRequestStream(vbno, opaqueMSB uint16, flags uint32,
vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error {
@ -793,7 +757,6 @@ func (feed *UprFeed) StartFeedWithConfig(datachan_len int) error {
}
func parseFailoverLog(body []byte) (*FailoverLog, error) {
if len(body)%16 != 0 {
err := fmt.Errorf("invalid body length %v, in failover-log", len(body))
return nil, err
@ -808,6 +771,24 @@ func parseFailoverLog(body []byte) (*FailoverLog, error) {
return &log, nil
}
func parseGetSeqnoResp(body []byte) (*VBSeqnos, error) {
// vbno of 2 bytes + seqno of 8 bytes
var entryLen int = 10
if len(body)%entryLen != 0 {
err := fmt.Errorf("invalid body length %v, in getVbSeqno", len(body))
return nil, err
}
vbSeqnos := make(VBSeqnos, len(body)/entryLen)
for i, j := 0, 0; i < len(body); i += entryLen {
vbno := binary.BigEndian.Uint16(body[i : i+2])
seqno := binary.BigEndian.Uint64(body[i+2 : i+10])
vbSeqnos[j] = [2]uint64{uint64(vbno), seqno}
j++
}
return &vbSeqnos, nil
}
func handleStreamRequest(
res *gomemcached.MCResponse,
headerBuf []byte,
@ -987,6 +968,14 @@ loop:
break loop
}
event = makeUprEvent(pkt, stream, bytes)
case gomemcached.UPR_FAILOVERLOG:
logging.Infof("Failover log for vb %d received: %v", vb, pkt)
case gomemcached.DCP_SEQNO_ADV:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
event = makeUprEvent(pkt, stream, bytes)
default:
logging.Infof("Recived an unknown response for vbucket %d", vb)
}

20
vendor/github.com/couchbase/gomemcached/mc_constants.go generated vendored
View file

@ -74,6 +74,7 @@ const (
TAP_VBUCKET_SET = CommandCode(0x45) // Sets state of vbucket in receiver (used in takeover)
TAP_CHECKPOINT_START = CommandCode(0x46) // Notifies start of new checkpoint
TAP_CHECKPOINT_END = CommandCode(0x47) // Notifies end of checkpoint
GET_ALL_VB_SEQNOS = CommandCode(0x48) // Get current high sequence numbers from all vbuckets located on the server
UPR_OPEN = CommandCode(0x50) // Open a UPR connection with a name
UPR_ADDSTREAM = CommandCode(0x51) // Sent by ebucketMigrator to UPR Consumer
@ -102,18 +103,21 @@ const (
SUBDOC_MULTI_LOOKUP = CommandCode(0xd0) // Multi lookup. Doc xattrs and meta.
DCP_SYSTEM_EVENT = CommandCode(0x5f) // A system event has occurred
DCP_SEQNO_ADV = CommandCode(0x64) // Sent when the vb seqno has advanced due to an unsubscribed event
)
// command codes that are counted toward DCP control buffer
// when DCP clients receive DCP messages with these command codes, they need to provide acknowledgement
var BufferedCommandCodeMap = map[CommandCode]bool{
SET_VBUCKET: true,
UPR_STREAMEND: true,
UPR_SNAPSHOT: true,
UPR_MUTATION: true,
UPR_DELETION: true,
UPR_EXPIRATION: true}
SET_VBUCKET: true,
UPR_STREAMEND: true,
UPR_SNAPSHOT: true,
UPR_MUTATION: true,
UPR_DELETION: true,
UPR_EXPIRATION: true,
DCP_SYSTEM_EVENT: true,
DCP_SEQNO_ADV: true,
}
// Status field for memcached response.
type Status uint16
@ -274,6 +278,8 @@ func init() {
CommandNames[SUBDOC_MULTI_LOOKUP] = "SUBDOC_MULTI_LOOKUP"
CommandNames[GET_COLLECTIONS_MANIFEST] = "GET_COLLECTIONS_MANIFEST"
CommandNames[COLLECTIONS_GET_CID] = "COLLECTIONS_GET_CID"
CommandNames[DCP_SYSTEM_EVENT] = "DCP_SYSTEM_EVENT"
CommandNames[DCP_SEQNO_ADV] = "DCP_SEQNO_ADV"
StatusNames = make(map[Status]string)
StatusNames[SUCCESS] = "SUCCESS"

61
vendor/github.com/couchbase/gomemcached/mc_req.go generated vendored
View file

@ -11,6 +11,8 @@ import (
// The current limit, 20MB, is the size limit supported by ep-engine.
var MaxBodyLen = int(20 * 1024 * 1024)
const _BUFLEN = 256
// MCRequest is memcached Request
type MCRequest struct {
// The command being issued
@ -27,6 +29,10 @@ type MCRequest struct {
DataType uint8
// len() calls are expensive - cache this in case for collection
Keylen int
// Collection id for collection based operations
CollId [binary.MaxVarintLen32]byte
// Length of collection id
CollIdLen int
// Flexible Framing Extras
FramingExtras []FrameInfo
// Stored length of incoming framing extras
@ -34,8 +40,12 @@ type MCRequest struct {
}
// Size gives the number of bytes this request requires.
func (req *MCRequest) HdrSize() int {
return HDR_LEN + len(req.Extras) + req.CollIdLen + req.FramingElen + len(req.Key)
}
func (req *MCRequest) Size() int {
return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta) + req.FramingElen
return req.HdrSize() + len(req.Body) + len(req.ExtMeta)
}
// A debugging string representation of this request
@ -68,7 +78,7 @@ func (req *MCRequest) fillRegularHeaderBytes(data []byte) int {
data[pos] = byte(req.Opcode)
pos++
binary.BigEndian.PutUint16(data[pos:pos+2],
uint16(len(req.Key)))
uint16(req.CollIdLen+len(req.Key)))
pos += 2
// 4
@ -84,7 +94,7 @@ func (req *MCRequest) fillRegularHeaderBytes(data []byte) int {
// 8
binary.BigEndian.PutUint32(data[pos:pos+4],
uint32(len(req.Body)+len(req.Key)+len(req.Extras)+len(req.ExtMeta)))
uint32(len(req.Body)+req.CollIdLen+len(req.Key)+len(req.Extras)+len(req.ExtMeta)))
pos += 4
// 12
@ -97,15 +107,21 @@ func (req *MCRequest) fillRegularHeaderBytes(data []byte) int {
}
pos += 8
// 24 - extras
if len(req.Extras) > 0 {
copy(data[pos:pos+len(req.Extras)], req.Extras)
pos += len(req.Extras)
}
if len(req.Key) > 0 {
if req.CollIdLen > 0 {
copy(data[pos:pos+req.CollIdLen], req.CollId[:])
pos += req.CollIdLen
}
copy(data[pos:pos+len(req.Key)], req.Key)
pos += len(req.Key)
}
return pos
}
@ -132,7 +148,7 @@ func (req *MCRequest) fillFlexHeaderBytes(data []byte) (int, bool) {
data[0] = FLEX_MAGIC
data[1] = byte(req.Opcode)
data[2] = byte(req.FramingElen)
data[3] = byte(req.Keylen)
data[3] = byte(req.Keylen + req.CollIdLen)
elen := len(req.Extras)
data[4] = byte(elen)
if req.DataType != 0 {
@ -140,7 +156,7 @@ func (req *MCRequest) fillFlexHeaderBytes(data []byte) (int, bool) {
}
binary.BigEndian.PutUint16(data[6:8], req.VBucket)
binary.BigEndian.PutUint32(data[8:12],
uint32(len(req.Body)+req.Keylen+elen+len(req.ExtMeta)+req.FramingElen))
uint32(len(req.Body)+req.Keylen+req.CollIdLen+elen+len(req.ExtMeta)+req.FramingElen))
binary.BigEndian.PutUint32(data[12:16], req.Opaque)
if req.Cas != 0 {
binary.BigEndian.PutUint64(data[16:24], req.Cas)
@ -205,12 +221,27 @@ func (req *MCRequest) fillFlexHeaderBytes(data []byte) (int, bool) {
// Add keys
if req.Keylen > 0 {
if mergeMode {
var key []byte
var keylen int
if req.CollIdLen == 0 {
key = req.Key
keylen = req.Keylen
} else {
key = append(key, req.CollId[:]...)
key = append(key, req.Key...)
keylen = req.Keylen + req.CollIdLen
}
outputBytes = ShiftByteSliceRight4Bits(req.Key)
data = Merge2HalfByteSlices(data, outputBytes)
pos += keylen
} else {
if req.CollIdLen > 0 {
copy(data[pos:pos+req.CollIdLen], req.CollId[:])
pos += req.CollIdLen
}
copy(data[pos:pos+req.Keylen], req.Key)
pos += req.Keylen
}
pos += req.Keylen
}
return pos, mergeMode
@ -227,7 +258,7 @@ func (req *MCRequest) FillHeaderBytes(data []byte) (int, bool) {
// HeaderBytes will return the wire representation of the request header
// (with the extras and key).
func (req *MCRequest) HeaderBytes() []byte {
data := make([]byte, HDR_LEN+len(req.Extras)+len(req.Key)+req.FramingElen)
data := make([]byte, HDR_LEN+len(req.Extras)+req.CollIdLen+len(req.Key)+req.FramingElen)
req.FillHeaderBytes(data)
@ -237,7 +268,11 @@ func (req *MCRequest) HeaderBytes() []byte {
// Bytes will return the wire representation of this request.
func (req *MCRequest) Bytes() []byte {
data := make([]byte, req.Size())
req.bytes(data)
return data
}
func (req *MCRequest) bytes(data []byte) {
pos, halfByteMode := req.FillHeaderBytes(data)
// TODO - the halfByteMode should be revisited for a more efficient
// way of doing things
@ -259,15 +294,19 @@ func (req *MCRequest) Bytes() []byte {
copy(data[pos+len(req.Body):pos+len(req.Body)+len(req.ExtMeta)], req.ExtMeta)
}
}
return data
}
// Transmit will send this request message across a writer.
func (req *MCRequest) Transmit(w io.Writer) (n int, err error) {
if len(req.Body) < 128 {
n, err = w.Write(req.Bytes())
l := req.Size()
if l < _BUFLEN {
data := make([]byte, l)
req.bytes(data)
n, err = w.Write(data)
} else {
n, err = w.Write(req.HeaderBytes())
data := make([]byte, req.HdrSize())
req.FillHeaderBytes(data)
n, err = w.Write(data)
if err == nil {
m := 0
m, err = w.Write(req.Body)