cmd/stdiscosrv: New discovery server (fixes #4618)
This is a new revision of the discovery server. Relevant changes and non-changes: - Protocol towards clients is unchanged. - Recommended large scale design is still to be deployed nehind nginx (I tested, and it's still a lot faster at terminating TLS). - Database backend is leveldb again, only. It scales enough, is easy to setup, and we don't need any backend to take care of. - Server supports replication. This is a simple TCP channel - protect it with a firewall when deploying over the internet. (We deploy this within the same datacenter, and with firewall.) Any incoming client announces are sent over the replication channel(s) to other peer discosrvs. Incoming replication changes are applied to the database as if they came from clients, but without the TLS/certificate overhead. - Metrics are exposed using the prometheus library, when enabled. - The database values and replication protocol is protobuf, because JSON was quite CPU intensive when I tried that and benchmarked it. - The "Retry-After" value for failed lookups gets slowly increased from a default of 120 seconds, by 5 seconds for each failed lookup, independently by each discosrv. This lowers the query load over time for clients that are never seen. The Retry-After maxes out at 3600 after a couple of weeks of this increase. The number of failed lookups is stored in the database, now and then (avoiding making each lookup a database put). All in all this means clients can be pointed towards a cluster using just multiple A / AAAA records to gain both load sharing and redundancy (if one is down, clients will talk to the remaining ones). GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4648
This commit is contained in:
Jakob Borg
306
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go
generated
vendored
Normal file
306
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go
generated
vendored
Normal file
@@ -0,0 +1,306 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3signer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Reference for constants used below -
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
|
||||
const (
|
||||
streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD"
|
||||
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
payloadChunkSize = 64 * 1024
|
||||
chunkSigConstLen = 17 // ";chunk-signature="
|
||||
signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2"
|
||||
crlfLen = 2 // CRLF
|
||||
)
|
||||
|
||||
// Request headers to be ignored while calculating seed signature for
|
||||
// a request.
|
||||
var ignoredStreamingHeaders = map[string]bool{
|
||||
"Authorization": true,
|
||||
"User-Agent": true,
|
||||
"Content-Type": true,
|
||||
}
|
||||
|
||||
// getSignedChunkLength - calculates the length of chunk metadata
|
||||
func getSignedChunkLength(chunkDataSize int64) int64 {
|
||||
return int64(len(fmt.Sprintf("%x", chunkDataSize))) +
|
||||
chunkSigConstLen +
|
||||
signatureStrLen +
|
||||
crlfLen +
|
||||
chunkDataSize +
|
||||
crlfLen
|
||||
}
|
||||
|
||||
// getStreamLength - calculates the length of the overall stream (data + metadata)
|
||||
func getStreamLength(dataLen, chunkSize int64) int64 {
|
||||
if dataLen <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
chunksCount := int64(dataLen / chunkSize)
|
||||
remainingBytes := int64(dataLen % chunkSize)
|
||||
streamLen := int64(0)
|
||||
streamLen += chunksCount * getSignedChunkLength(chunkSize)
|
||||
if remainingBytes > 0 {
|
||||
streamLen += getSignedChunkLength(remainingBytes)
|
||||
}
|
||||
streamLen += getSignedChunkLength(0)
|
||||
return streamLen
|
||||
}
|
||||
|
||||
// buildChunkStringToSign - returns the string to sign given chunk data
|
||||
// and previous signature.
|
||||
func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
|
||||
stringToSignParts := []string{
|
||||
streamingPayloadHdr,
|
||||
t.Format(iso8601DateFormat),
|
||||
getScope(region, t),
|
||||
previousSig,
|
||||
emptySHA256,
|
||||
hex.EncodeToString(sum256(chunkData)),
|
||||
}
|
||||
|
||||
return strings.Join(stringToSignParts, "\n")
|
||||
}
|
||||
|
||||
// prepareStreamingRequest - prepares a request with appropriate
|
||||
// headers before computing the seed signature.
|
||||
func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) {
|
||||
// Set x-amz-content-sha256 header.
|
||||
req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm)
|
||||
if sessionToken != "" {
|
||||
req.Header.Set("X-Amz-Security-Token", sessionToken)
|
||||
}
|
||||
|
||||
req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat))
|
||||
// Set content length with streaming signature for each chunk included.
|
||||
req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize))
|
||||
req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10))
|
||||
}
|
||||
|
||||
// buildChunkHeader - returns the chunk header.
|
||||
// e.g string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
|
||||
func buildChunkHeader(chunkLen int64, signature string) []byte {
|
||||
return []byte(strconv.FormatInt(chunkLen, 16) + ";chunk-signature=" + signature + "\r\n")
|
||||
}
|
||||
|
||||
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
|
||||
func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
|
||||
previousSignature, secretAccessKey string) string {
|
||||
|
||||
chunkStringToSign := buildChunkStringToSign(reqTime, region,
|
||||
previousSignature, chunkData)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime)
|
||||
return getSignature(signingKey, chunkStringToSign)
|
||||
}
|
||||
|
||||
// getSeedSignature - returns the seed signature for a given request.
|
||||
func (s *StreamingReader) setSeedSignature(req *http.Request) {
|
||||
// Get canonical request
|
||||
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders)
|
||||
|
||||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest)
|
||||
|
||||
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime)
|
||||
|
||||
// Calculate signature.
|
||||
s.seedSignature = getSignature(signingKey, stringToSign)
|
||||
}
|
||||
|
||||
// StreamingReader implements chunked upload signature as a reader on
|
||||
// top of req.Body's ReaderCloser chunk header;data;... repeat
|
||||
type StreamingReader struct {
|
||||
accessKeyID string
|
||||
secretAccessKey string
|
||||
sessionToken string
|
||||
region string
|
||||
prevSignature string
|
||||
seedSignature string
|
||||
contentLen int64 // Content-Length from req header
|
||||
baseReadCloser io.ReadCloser // underlying io.Reader
|
||||
bytesRead int64 // bytes read from underlying io.Reader
|
||||
buf bytes.Buffer // holds signed chunk
|
||||
chunkBuf []byte // holds raw data read from req Body
|
||||
chunkBufLen int // no. of bytes read so far into chunkBuf
|
||||
done bool // done reading the underlying reader to EOF
|
||||
reqTime time.Time
|
||||
chunkNum int
|
||||
totalChunks int
|
||||
lastChunkSize int
|
||||
}
|
||||
|
||||
// signChunk - signs a chunk read from s.baseReader of chunkLen size.
|
||||
func (s *StreamingReader) signChunk(chunkLen int) {
|
||||
// Compute chunk signature for next header
|
||||
signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime,
|
||||
s.region, s.prevSignature, s.secretAccessKey)
|
||||
|
||||
// For next chunk signature computation
|
||||
s.prevSignature = signature
|
||||
|
||||
// Write chunk header into streaming buffer
|
||||
chunkHdr := buildChunkHeader(int64(chunkLen), signature)
|
||||
s.buf.Write(chunkHdr)
|
||||
|
||||
// Write chunk data into streaming buffer
|
||||
s.buf.Write(s.chunkBuf[:chunkLen])
|
||||
|
||||
// Write the chunk trailer.
|
||||
s.buf.Write([]byte("\r\n"))
|
||||
|
||||
// Reset chunkBufLen for next chunk read.
|
||||
s.chunkBufLen = 0
|
||||
s.chunkNum++
|
||||
}
|
||||
|
||||
// setStreamingAuthHeader - builds and sets authorization header value
|
||||
// for streaming signature.
|
||||
func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
|
||||
credential := GetCredential(s.accessKeyID, s.region, s.reqTime)
|
||||
authParts := []string{
|
||||
signV4Algorithm + " Credential=" + credential,
|
||||
"SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders),
|
||||
"Signature=" + s.seedSignature,
|
||||
}
|
||||
|
||||
// Set authorization header.
|
||||
auth := strings.Join(authParts, ",")
|
||||
req.Header.Set("Authorization", auth)
|
||||
}
|
||||
|
||||
// StreamingSignV4 - provides chunked upload signatureV4 support by
|
||||
// implementing io.Reader.
|
||||
func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
|
||||
region string, dataLen int64, reqTime time.Time) *http.Request {
|
||||
|
||||
// Set headers needed for streaming signature.
|
||||
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
|
||||
|
||||
if req.Body == nil {
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
|
||||
}
|
||||
|
||||
stReader := &StreamingReader{
|
||||
baseReadCloser: req.Body,
|
||||
accessKeyID: accessKeyID,
|
||||
secretAccessKey: secretAccessKey,
|
||||
sessionToken: sessionToken,
|
||||
region: region,
|
||||
reqTime: reqTime,
|
||||
chunkBuf: make([]byte, payloadChunkSize),
|
||||
contentLen: dataLen,
|
||||
chunkNum: 1,
|
||||
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
|
||||
lastChunkSize: int(dataLen % payloadChunkSize),
|
||||
}
|
||||
|
||||
// Add the request headers required for chunk upload signing.
|
||||
|
||||
// Compute the seed signature.
|
||||
stReader.setSeedSignature(req)
|
||||
|
||||
// Set the authorization header with the seed signature.
|
||||
stReader.setStreamingAuthHeader(req)
|
||||
|
||||
// Set seed signature as prevSignature for subsequent
|
||||
// streaming signing process.
|
||||
stReader.prevSignature = stReader.seedSignature
|
||||
req.Body = stReader
|
||||
|
||||
return req
|
||||
}
|
||||
|
||||
// Read - this method performs chunk upload signature providing a
|
||||
// io.Reader interface.
|
||||
func (s *StreamingReader) Read(buf []byte) (int, error) {
|
||||
switch {
|
||||
// After the last chunk is read from underlying reader, we
|
||||
// never re-fill s.buf.
|
||||
case s.done:
|
||||
|
||||
// s.buf will be (re-)filled with next chunk when has lesser
|
||||
// bytes than asked for.
|
||||
case s.buf.Len() < len(buf):
|
||||
s.chunkBufLen = 0
|
||||
for {
|
||||
n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:])
|
||||
// Usually we validate `err` first, but in this case
|
||||
// we are validating n > 0 for the following reasons.
|
||||
//
|
||||
// 1. n > 0, err is one of io.EOF, nil (near end of stream)
|
||||
// A Reader returning a non-zero number of bytes at the end
|
||||
// of the input stream may return either err == EOF or err == nil
|
||||
//
|
||||
// 2. n == 0, err is io.EOF (actual end of stream)
|
||||
//
|
||||
// Callers should always process the n > 0 bytes returned
|
||||
// before considering the error err.
|
||||
if n1 > 0 {
|
||||
s.chunkBufLen += n1
|
||||
s.bytesRead += int64(n1)
|
||||
|
||||
if s.chunkBufLen == payloadChunkSize ||
|
||||
(s.chunkNum == s.totalChunks-1 &&
|
||||
s.chunkBufLen == s.lastChunkSize) {
|
||||
// Sign the chunk and write it to s.buf.
|
||||
s.signChunk(s.chunkBufLen)
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// No more data left in baseReader - last chunk.
|
||||
// Done reading the last chunk from baseReader.
|
||||
s.done = true
|
||||
|
||||
// bytes read from baseReader different than
|
||||
// content length provided.
|
||||
if s.bytesRead != s.contentLen {
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// Sign the chunk and write it to s.buf.
|
||||
s.signChunk(0)
|
||||
break
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return s.buf.Read(buf)
|
||||
}
|
||||
|
||||
// Close - this method makes underlying io.ReadCloser's Close method available.
|
||||
func (s *StreamingReader) Close() error {
|
||||
return s.baseReadCloser.Close()
|
||||
}
|
||||
320
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go
generated
vendored
Normal file
320
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go
generated
vendored
Normal file
@@ -0,0 +1,320 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2015-2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3signer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
)
|
||||
|
||||
// Signature and API related constants.
|
||||
const (
|
||||
signV2Algorithm = "AWS"
|
||||
)
|
||||
|
||||
// Encode input URL path to URL encoded path.
|
||||
func encodeURL2Path(u *url.URL) (path string) {
|
||||
// Encode URL path.
|
||||
if isS3, _ := filepath.Match("*.s3*.amazonaws.com", u.Host); isS3 {
|
||||
bucketName := u.Host[:strings.LastIndex(u.Host, ".s3")]
|
||||
path = "/" + bucketName
|
||||
path += u.Path
|
||||
path = s3utils.EncodePath(path)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(u.Host, ".storage.googleapis.com") {
|
||||
path = "/" + strings.TrimSuffix(u.Host, ".storage.googleapis.com")
|
||||
path += u.Path
|
||||
path = s3utils.EncodePath(path)
|
||||
return
|
||||
}
|
||||
path = s3utils.EncodePath(u.Path)
|
||||
return
|
||||
}
|
||||
|
||||
// PreSignV2 - presign the request in following style.
|
||||
// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
|
||||
func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64) *http.Request {
|
||||
// Presign is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
d := time.Now().UTC()
|
||||
// Find epoch expires when the request will expire.
|
||||
epochExpires := d.Unix() + expires
|
||||
|
||||
// Add expires header if not present.
|
||||
if expiresStr := req.Header.Get("Expires"); expiresStr == "" {
|
||||
req.Header.Set("Expires", strconv.FormatInt(epochExpires, 10))
|
||||
}
|
||||
|
||||
// Get presigned string to sign.
|
||||
stringToSign := preStringToSignV2(req)
|
||||
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||
hm.Write([]byte(stringToSign))
|
||||
|
||||
// Calculate signature.
|
||||
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
||||
|
||||
query := req.URL.Query()
|
||||
// Handle specially for Google Cloud Storage.
|
||||
if strings.Contains(req.URL.Host, ".storage.googleapis.com") {
|
||||
query.Set("GoogleAccessId", accessKeyID)
|
||||
} else {
|
||||
query.Set("AWSAccessKeyId", accessKeyID)
|
||||
}
|
||||
|
||||
// Fill in Expires for presigned query.
|
||||
query.Set("Expires", strconv.FormatInt(epochExpires, 10))
|
||||
|
||||
// Encode query and save.
|
||||
req.URL.RawQuery = s3utils.QueryEncode(query)
|
||||
|
||||
// Save signature finally.
|
||||
req.URL.RawQuery += "&Signature=" + s3utils.EncodePath(signature)
|
||||
|
||||
// Return.
|
||||
return &req
|
||||
}
|
||||
|
||||
// PostPresignSignatureV2 - presigned signature for PostPolicy
|
||||
// request.
|
||||
func PostPresignSignatureV2(policyBase64, secretAccessKey string) string {
|
||||
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||
hm.Write([]byte(policyBase64))
|
||||
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
||||
return signature
|
||||
}
|
||||
|
||||
// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
|
||||
// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
|
||||
//
|
||||
// StringToSign = HTTP-Verb + "\n" +
|
||||
// Content-Md5 + "\n" +
|
||||
// Content-Type + "\n" +
|
||||
// Date + "\n" +
|
||||
// CanonicalizedProtocolHeaders +
|
||||
// CanonicalizedResource;
|
||||
//
|
||||
// CanonicalizedResource = [ "/" + Bucket ] +
|
||||
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
||||
// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
||||
//
|
||||
// CanonicalizedProtocolHeaders = <described below>
|
||||
|
||||
// SignV2 sign the request before Do() (AWS Signature Version 2).
|
||||
func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request {
|
||||
// Signature calculation is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
d := time.Now().UTC()
|
||||
|
||||
// Add date if not present.
|
||||
if date := req.Header.Get("Date"); date == "" {
|
||||
req.Header.Set("Date", d.Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
// Calculate HMAC for secretAccessKey.
|
||||
stringToSign := stringToSignV2(req)
|
||||
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||
hm.Write([]byte(stringToSign))
|
||||
|
||||
// Prepare auth header.
|
||||
authHeader := new(bytes.Buffer)
|
||||
authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID))
|
||||
encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
|
||||
encoder.Write(hm.Sum(nil))
|
||||
encoder.Close()
|
||||
|
||||
// Set Authorization header.
|
||||
req.Header.Set("Authorization", authHeader.String())
|
||||
|
||||
return &req
|
||||
}
|
||||
|
||||
// From the Amazon docs:
|
||||
//
|
||||
// StringToSign = HTTP-Verb + "\n" +
|
||||
// Content-Md5 + "\n" +
|
||||
// Content-Type + "\n" +
|
||||
// Expires + "\n" +
|
||||
// CanonicalizedProtocolHeaders +
|
||||
// CanonicalizedResource;
|
||||
func preStringToSignV2(req http.Request) string {
|
||||
buf := new(bytes.Buffer)
|
||||
// Write standard headers.
|
||||
writePreSignV2Headers(buf, req)
|
||||
// Write canonicalized protocol headers if any.
|
||||
writeCanonicalizedHeaders(buf, req)
|
||||
// Write canonicalized Query resources if any.
|
||||
writeCanonicalizedResource(buf, req)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// writePreSignV2Headers - write preSign v2 required headers.
|
||||
func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
|
||||
buf.WriteString(req.Method + "\n")
|
||||
buf.WriteString(req.Header.Get("Content-Md5") + "\n")
|
||||
buf.WriteString(req.Header.Get("Content-Type") + "\n")
|
||||
buf.WriteString(req.Header.Get("Expires") + "\n")
|
||||
}
|
||||
|
||||
// From the Amazon docs:
|
||||
//
|
||||
// StringToSign = HTTP-Verb + "\n" +
|
||||
// Content-Md5 + "\n" +
|
||||
// Content-Type + "\n" +
|
||||
// Date + "\n" +
|
||||
// CanonicalizedProtocolHeaders +
|
||||
// CanonicalizedResource;
|
||||
func stringToSignV2(req http.Request) string {
|
||||
buf := new(bytes.Buffer)
|
||||
// Write standard headers.
|
||||
writeSignV2Headers(buf, req)
|
||||
// Write canonicalized protocol headers if any.
|
||||
writeCanonicalizedHeaders(buf, req)
|
||||
// Write canonicalized Query resources if any.
|
||||
writeCanonicalizedResource(buf, req)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// writeSignV2Headers - write signV2 required headers.
|
||||
func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
|
||||
buf.WriteString(req.Method + "\n")
|
||||
buf.WriteString(req.Header.Get("Content-Md5") + "\n")
|
||||
buf.WriteString(req.Header.Get("Content-Type") + "\n")
|
||||
buf.WriteString(req.Header.Get("Date") + "\n")
|
||||
}
|
||||
|
||||
// writeCanonicalizedHeaders - write canonicalized headers.
|
||||
func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
|
||||
var protoHeaders []string
|
||||
vals := make(map[string][]string)
|
||||
for k, vv := range req.Header {
|
||||
// All the AMZ headers should be lowercase
|
||||
lk := strings.ToLower(k)
|
||||
if strings.HasPrefix(lk, "x-amz") {
|
||||
protoHeaders = append(protoHeaders, lk)
|
||||
vals[lk] = vv
|
||||
}
|
||||
}
|
||||
sort.Strings(protoHeaders)
|
||||
for _, k := range protoHeaders {
|
||||
buf.WriteString(k)
|
||||
buf.WriteByte(':')
|
||||
for idx, v := range vals[k] {
|
||||
if idx > 0 {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
if strings.Contains(v, "\n") {
|
||||
// TODO: "Unfold" long headers that
|
||||
// span multiple lines (as allowed by
|
||||
// RFC 2616, section 4.2) by replacing
|
||||
// the folding white-space (including
|
||||
// new-line) by a single space.
|
||||
buf.WriteString(v)
|
||||
} else {
|
||||
buf.WriteString(v)
|
||||
}
|
||||
}
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
// AWS S3 Signature V2 calculation rule is give here:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign
|
||||
|
||||
// Whitelist resource list that will be used in query string for signature-V2 calculation.
|
||||
// The list should be alphabetically sorted
|
||||
var resourceList = []string{
|
||||
"acl",
|
||||
"delete",
|
||||
"lifecycle",
|
||||
"location",
|
||||
"logging",
|
||||
"notification",
|
||||
"partNumber",
|
||||
"policy",
|
||||
"requestPayment",
|
||||
"response-cache-control",
|
||||
"response-content-disposition",
|
||||
"response-content-encoding",
|
||||
"response-content-language",
|
||||
"response-content-type",
|
||||
"response-expires",
|
||||
"torrent",
|
||||
"uploadId",
|
||||
"uploads",
|
||||
"versionId",
|
||||
"versioning",
|
||||
"versions",
|
||||
"website",
|
||||
}
|
||||
|
||||
// From the Amazon docs:
|
||||
//
|
||||
// CanonicalizedResource = [ "/" + Bucket ] +
|
||||
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
||||
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
||||
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) {
|
||||
// Save request URL.
|
||||
requestURL := req.URL
|
||||
// Get encoded URL path.
|
||||
buf.WriteString(encodeURL2Path(requestURL))
|
||||
if requestURL.RawQuery != "" {
|
||||
var n int
|
||||
vals, _ := url.ParseQuery(requestURL.RawQuery)
|
||||
// Verify if any sub resource queries are present, if yes
|
||||
// canonicallize them.
|
||||
for _, resource := range resourceList {
|
||||
if vv, ok := vals[resource]; ok && len(vv) > 0 {
|
||||
n++
|
||||
// First element
|
||||
switch n {
|
||||
case 1:
|
||||
buf.WriteByte('?')
|
||||
// The rest
|
||||
default:
|
||||
buf.WriteByte('&')
|
||||
}
|
||||
buf.WriteString(resource)
|
||||
// Request parameters
|
||||
if len(vv[0]) > 0 {
|
||||
buf.WriteByte('=')
|
||||
buf.WriteString(vv[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
315
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go
generated
vendored
Normal file
315
vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go
generated
vendored
Normal file
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2015-2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3signer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
)
|
||||
|
||||
// Signature and API related constants.
|
||||
const (
|
||||
signV4Algorithm = "AWS4-HMAC-SHA256"
|
||||
iso8601DateFormat = "20060102T150405Z"
|
||||
yyyymmdd = "20060102"
|
||||
)
|
||||
|
||||
///
|
||||
/// Excerpts from @lsegal -
|
||||
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
|
||||
///
|
||||
/// User-Agent:
|
||||
///
|
||||
/// This is ignored from signing because signing this causes
|
||||
/// problems with generating pre-signed URLs (that are executed
|
||||
/// by other agents) or when customers pass requests through
|
||||
/// proxies, which may modify the user-agent.
|
||||
///
|
||||
/// Content-Length:
|
||||
///
|
||||
/// This is ignored from signing because generating a pre-signed
|
||||
/// URL should not provide a content-length constraint,
|
||||
/// specifically when vending a S3 pre-signed PUT URL. The
|
||||
/// corollary to this is that when sending regular requests
|
||||
/// (non-pre-signed), the signature contains a checksum of the
|
||||
/// body, which implicitly validates the payload length (since
|
||||
/// changing the number of bytes would change the checksum)
|
||||
/// and therefore this header is not valuable in the signature.
|
||||
///
|
||||
/// Content-Type:
|
||||
///
|
||||
/// Signing this header causes quite a number of problems in
|
||||
/// browser environments, where browsers like to modify and
|
||||
/// normalize the content-type header in different ways. There is
|
||||
/// more information on this in https://goo.gl/2E9gyy. Avoiding
|
||||
/// this field simplifies logic and reduces the possibility of
|
||||
/// future bugs.
|
||||
///
|
||||
/// Authorization:
|
||||
///
|
||||
/// Is skipped for obvious reasons
|
||||
///
|
||||
var v4IgnoredHeaders = map[string]bool{
|
||||
"Authorization": true,
|
||||
"Content-Type": true,
|
||||
"Content-Length": true,
|
||||
"User-Agent": true,
|
||||
}
|
||||
|
||||
// getSigningKey hmac seed to calculate final signature.
|
||||
func getSigningKey(secret, loc string, t time.Time) []byte {
|
||||
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
|
||||
location := sumHMAC(date, []byte(loc))
|
||||
service := sumHMAC(location, []byte("s3"))
|
||||
signingKey := sumHMAC(service, []byte("aws4_request"))
|
||||
return signingKey
|
||||
}
|
||||
|
||||
// getSignature final signature in hexadecimal form.
|
||||
func getSignature(signingKey []byte, stringToSign string) string {
|
||||
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
|
||||
}
|
||||
|
||||
// getScope generate a string of a specific date, an AWS region, and a
|
||||
// service.
|
||||
func getScope(location string, t time.Time) string {
|
||||
scope := strings.Join([]string{
|
||||
t.Format(yyyymmdd),
|
||||
location,
|
||||
"s3",
|
||||
"aws4_request",
|
||||
}, "/")
|
||||
return scope
|
||||
}
|
||||
|
||||
// GetCredential generate a credential string.
|
||||
func GetCredential(accessKeyID, location string, t time.Time) string {
|
||||
scope := getScope(location, t)
|
||||
return accessKeyID + "/" + scope
|
||||
}
|
||||
|
||||
// getHashedPayload get the hexadecimal value of the SHA256 hash of
|
||||
// the request payload.
|
||||
func getHashedPayload(req http.Request) string {
|
||||
hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
|
||||
if hashedPayload == "" {
|
||||
// Presign does not have a payload, use S3 recommended value.
|
||||
hashedPayload = unsignedPayload
|
||||
}
|
||||
return hashedPayload
|
||||
}
|
||||
|
||||
// getCanonicalHeaders generate a list of request headers for
|
||||
// signature.
|
||||
func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string {
|
||||
var headers []string
|
||||
vals := make(map[string][]string)
|
||||
for k, vv := range req.Header {
|
||||
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
|
||||
continue // ignored header
|
||||
}
|
||||
headers = append(headers, strings.ToLower(k))
|
||||
vals[strings.ToLower(k)] = vv
|
||||
}
|
||||
headers = append(headers, "host")
|
||||
sort.Strings(headers)
|
||||
|
||||
var buf bytes.Buffer
|
||||
// Save all the headers in canonical form <header>:<value> newline
|
||||
// separated for each header.
|
||||
for _, k := range headers {
|
||||
buf.WriteString(k)
|
||||
buf.WriteByte(':')
|
||||
switch {
|
||||
case k == "host":
|
||||
buf.WriteString(req.URL.Host)
|
||||
fallthrough
|
||||
default:
|
||||
for idx, v := range vals[k] {
|
||||
if idx > 0 {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
buf.WriteString(v)
|
||||
}
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// getSignedHeaders generate all signed request headers.
|
||||
// i.e lexically sorted, semicolon-separated list of lowercase
|
||||
// request header names.
|
||||
func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string {
|
||||
var headers []string
|
||||
for k := range req.Header {
|
||||
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
|
||||
continue // Ignored header found continue.
|
||||
}
|
||||
headers = append(headers, strings.ToLower(k))
|
||||
}
|
||||
headers = append(headers, "host")
|
||||
sort.Strings(headers)
|
||||
return strings.Join(headers, ";")
|
||||
}
|
||||
|
||||
// getCanonicalRequest generate a canonical request of style.
|
||||
//
|
||||
// canonicalRequest =
|
||||
// <HTTPMethod>\n
|
||||
// <CanonicalURI>\n
|
||||
// <CanonicalQueryString>\n
|
||||
// <CanonicalHeaders>\n
|
||||
// <SignedHeaders>\n
|
||||
// <HashedPayload>
|
||||
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string {
|
||||
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
|
||||
canonicalRequest := strings.Join([]string{
|
||||
req.Method,
|
||||
s3utils.EncodePath(req.URL.Path),
|
||||
req.URL.RawQuery,
|
||||
getCanonicalHeaders(req, ignoredHeaders),
|
||||
getSignedHeaders(req, ignoredHeaders),
|
||||
getHashedPayload(req),
|
||||
}, "\n")
|
||||
return canonicalRequest
|
||||
}
|
||||
|
||||
// getStringToSign a string based on selected query values.
|
||||
func getStringToSignV4(t time.Time, location, canonicalRequest string) string {
|
||||
stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
|
||||
stringToSign = stringToSign + getScope(location, t) + "\n"
|
||||
stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
|
||||
return stringToSign
|
||||
}
|
||||
|
||||
// PreSignV4 presign the request, in accordance with
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
|
||||
func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
|
||||
// Presign is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
t := time.Now().UTC()
|
||||
|
||||
// Get credential string.
|
||||
credential := GetCredential(accessKeyID, location, t)
|
||||
|
||||
// Get all signed headers.
|
||||
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
|
||||
|
||||
// Set URL query.
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Algorithm", signV4Algorithm)
|
||||
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
|
||||
query.Set("X-Amz-SignedHeaders", signedHeaders)
|
||||
query.Set("X-Amz-Credential", credential)
|
||||
// Set session token if available.
|
||||
if sessionToken != "" {
|
||||
query.Set("X-Amz-Security-Token", sessionToken)
|
||||
}
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
// Get canonical request.
|
||||
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders)
|
||||
|
||||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||
|
||||
// Gext hmac signing key.
|
||||
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
|
||||
// Add signature header to RawQuery.
|
||||
req.URL.RawQuery += "&X-Amz-Signature=" + signature
|
||||
|
||||
return &req
|
||||
}
|
||||
|
||||
// PostPresignSignatureV4 - presigned signature for PostPolicy
|
||||
// requests.
|
||||
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
|
||||
// Get signining key.
|
||||
signingkey := getSigningKey(secretAccessKey, location, t)
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingkey, policyBase64)
|
||||
return signature
|
||||
}
|
||||
|
||||
// SignV4 sign the request before Do(), in accordance with
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
|
||||
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
|
||||
// Signature calculation is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
// Initial time.
|
||||
t := time.Now().UTC()
|
||||
|
||||
// Set x-amz-date.
|
||||
req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||
|
||||
// Set session token if available.
|
||||
if sessionToken != "" {
|
||||
req.Header.Set("X-Amz-Security-Token", sessionToken)
|
||||
}
|
||||
|
||||
// Get canonical request.
|
||||
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders)
|
||||
|
||||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||
|
||||
// Get hmac signing key.
|
||||
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||
|
||||
// Get credential string.
|
||||
credential := GetCredential(accessKeyID, location, t)
|
||||
|
||||
// Get all signed headers.
|
||||
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
|
||||
|
||||
// Calculate signature.
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
|
||||
// If regular request, construct the final authorization header.
|
||||
parts := []string{
|
||||
signV4Algorithm + " Credential=" + credential,
|
||||
"SignedHeaders=" + signedHeaders,
|
||||
"Signature=" + signature,
|
||||
}
|
||||
|
||||
// Set authorization header.
|
||||
auth := strings.Join(parts, ", ")
|
||||
req.Header.Set("Authorization", auth)
|
||||
|
||||
return &req
|
||||
}
|
||||
40
vendor/github.com/minio/minio-go/pkg/s3signer/utils.go
generated
vendored
Normal file
40
vendor/github.com/minio/minio-go/pkg/s3signer/utils.go
generated
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2015-2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3signer
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
)
|
||||
|
||||
// unsignedPayload - value to be set to X-Amz-Content-Sha256 header when
|
||||
const unsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
|
||||
// sum256 calculate sha256 sum for an input byte array.
|
||||
func sum256(data []byte) []byte {
|
||||
hash := sha256.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// sumHMAC calculate hmac between two input byte array.
|
||||
func sumHMAC(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
Reference in New Issue
Block a user