nune/gitea
nune/gitea
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Azure blob storage support (#30995)

Browse source 
This PR implemented object storages(LFS/Packages/Attachments and etc.)
for Azure Blob Storage. It depends on azure official golang SDK and can
support both the azure blob storage cloud service and azurite mock
server.

Replace #25458
Fix #22527 

- [x] CI Tests
- [x] integration test, MSSQL integration tests will now based on
azureblob
  - [x] unit test 
- [x] CLI Migrate Storage
- [x] Documentation for configuration added

------

TODO (other PRs):
- [ ] Improve performance of `blob download`.

---------

Co-authored-by: yp05327 <576951401@qq.com>
This commit is contained in:
Lunny Xiao 2024-05-30 15:33:50 +08:00 • committed by GitHub
parent 015efcd8bf
commit fb7b743bd0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 779 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

90
modules/setting/storage.go
View file

@ -18,11 +18,14 @@ const (
LocalStorageType StorageType = "local"
// MinioStorageType is the type descriptor for minio storage
MinioStorageType StorageType = "minio"
// AzureBlobStorageType is the type descriptor for azure blob storage
AzureBlobStorageType StorageType = "azureblob"
)
var storageTypes = []StorageType{
LocalStorageType,
MinioStorageType,
AzureBlobStorageType,
}
// IsValidStorageType returns true if the given storage type is valid
@ -50,25 +53,55 @@ type MinioStorageConfig struct {
BucketLookUpType string `ini:"MINIO_BUCKET_LOOKUP_TYPE" json:",omitempty"`
}
func (cfg *MinioStorageConfig) ToShadow() {
if cfg.AccessKeyID != "" {
cfg.AccessKeyID = "******"
}
if cfg.SecretAccessKey != "" {
cfg.SecretAccessKey = "******"
}
}
// MinioStorageConfig represents the configuration for a minio storage
type AzureBlobStorageConfig struct {
Endpoint string `ini:"AZURE_BLOB_ENDPOINT" json:",omitempty"`
AccountName string `ini:"AZURE_BLOB_ACCOUNT_NAME" json:",omitempty"`
AccountKey string `ini:"AZURE_BLOB_ACCOUNT_KEY" json:",omitempty"`
Container string `ini:"AZURE_BLOB_CONTAINER" json:",omitempty"`
BasePath string `ini:"AZURE_BLOB_BASE_PATH" json:",omitempty"`
ServeDirect bool `ini:"SERVE_DIRECT"`
}
func (cfg *AzureBlobStorageConfig) ToShadow() {
if cfg.AccountKey != "" {
cfg.AccountKey = "******"
}
if cfg.AccountName != "" {
cfg.AccountName = "******"
}
}
// Storage represents configuration of storages
type Storage struct {
Type StorageType // local or minio
Path string `json:",omitempty"` // for local type
TemporaryPath string `json:",omitempty"`
MinioConfig MinioStorageConfig // for minio type
Type StorageType // local or minio or azureblob
Path string `json:",omitempty"` // for local type
TemporaryPath string `json:",omitempty"`
MinioConfig MinioStorageConfig // for minio type
AzureBlobConfig AzureBlobStorageConfig // for azureblob type
}
func (storage *Storage) ToShadowCopy() Storage {
shadowStorage := *storage
if shadowStorage.MinioConfig.AccessKeyID != "" {
shadowStorage.MinioConfig.AccessKeyID = "******"
}
if shadowStorage.MinioConfig.SecretAccessKey != "" {
shadowStorage.MinioConfig.SecretAccessKey = "******"
}
shadowStorage.MinioConfig.ToShadow()
shadowStorage.AzureBlobConfig.ToShadow()
return shadowStorage
}
func (storage *Storage) ServeDirect() bool {
return (storage.Type == MinioStorageType && storage.MinioConfig.ServeDirect) ||
(storage.Type == AzureBlobStorageType && storage.AzureBlobConfig.ServeDirect)
}
const storageSectionName = "storage"
func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
@ -84,6 +117,10 @@ func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
storageSec.Key("MINIO_BUCKET_LOOKUP_TYPE").MustString("auto")
storageSec.Key("AZURE_BLOB_ENDPOINT").MustString("")
storageSec.Key("AZURE_BLOB_ACCOUNT_NAME").MustString("")
storageSec.Key("AZURE_BLOB_ACCOUNT_KEY").MustString("")
storageSec.Key("AZURE_BLOB_CONTAINER").MustString("gitea")
return storageSec
}
@ -107,6 +144,8 @@ func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*S
return getStorageForLocal(targetSec, overrideSec, tp, name)
case string(MinioStorageType):
return getStorageForMinio(targetSec, overrideSec, tp, name)
case string(AzureBlobStorageType):
return getStorageForAzureBlob(targetSec, overrideSec, tp, name)
default:
return nil, fmt.Errorf("unsupported storage type %q", targetType)
}
@ -247,7 +286,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType,
return &storage, nil
}
func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) {
func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl
var storage Storage
storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String())
if err := targetSec.MapTo(&storage.MinioConfig); err != nil {
@ -275,3 +314,32 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType,
}
return &storage, nil
}
func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl
var storage Storage
storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String())
if err := targetSec.MapTo(&storage.AzureBlobConfig); err != nil {
return nil, fmt.Errorf("map azure blob config failed: %v", err)
}
var defaultPath string
if storage.AzureBlobConfig.BasePath != "" {
if tp == targetSecIsStorage || tp == targetSecIsDefault {
defaultPath = strings.TrimSuffix(storage.AzureBlobConfig.BasePath, "/") + "/" + name + "/"
} else {
defaultPath = storage.AzureBlobConfig.BasePath
}
}
if defaultPath == "" {
defaultPath = name + "/"
}
if overrideSec != nil {
storage.AzureBlobConfig.ServeDirect = ConfigSectionKeyBool(overrideSec, "SERVE_DIRECT", storage.AzureBlobConfig.ServeDirect)
storage.AzureBlobConfig.BasePath = ConfigSectionKeyString(overrideSec, "AZURE_BLOB_BASE_PATH", defaultPath)
storage.AzureBlobConfig.Container = ConfigSectionKeyString(overrideSec, "AZURE_BLOB_CONTAINER", storage.AzureBlobConfig.Container)
} else {
storage.AzureBlobConfig.BasePath = defaultPath
}
return &storage, nil
}