10000 Azure blob storage support by lunny · Pull Request #30995 · go-gitea/gitea · GitHub
[go: up one dir, main page]

Skip to content

Azure blob storage support #30995

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 56 commits into from
May 30, 2024
Merged
Changes from 1 commit
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
0a0c188
first
yp05327 Jun 23, 2023
fce996a
fix lint
yp05327 Jun 23, 2023
efc0ed7
add ci test
yp05327 Jun 23, 2023
f3d994d
Merge branch 'main' into azure-blob-storage-support
yp05327 Jun 23, 2023
888cfa5
fix url
yp05327 Jun 23, 2023
ab798ec
remove
yp05327 Jul 6, 2023
48d98e7
add todo
yp05327 Jul 6, 2023
3901076
Merge branch 'main' into azure-blob-storage-support
yp05327 Jul 25, 2023
ec57c06
Merge branch 'main' into azure-blob-storage-support
yp05327 Nov 17, 2023
2c00368
update azure sdk
yp05327 Nov 17, 2023
f45367f
update
yp05327 Nov 17, 2023
e03829f
rename to azureblob
yp05327 Nov 17, 2023
5ade526
support direct url for azure blob
yp05327 Nov 17, 2023
3b49d30
fix
yp05327 Nov 17, 2023
bc27561
improve
yp05327 Nov 20, 2023
e438513
imporve
yp05327 Nov 20, 2023
8e007dc
improve azure blob url
yp05327 Nov 24, 2023
5faecb9
add path test
yp05327 Nov 24, 2023
f7e42d2
fix lint
yp05327 Nov 24, 2023
2a8add1
Merge branch 'azure-blob-storage-support' of github.com:yp05327/gitea…
lunny May 2, 2024
14a194c
Add some tests for configuration of azure blob storage support
lunny May 2, 2024
c6e3538
upgrade azure blob version
lunny May 6, 2024
594cbeb
Merge b 8000 ranch 'main' into lunny/azure_blob_storage
lunny May 11, 2024
95ab4d1
Fix download blob
lunny May 13, 2024
de67cea
Fix bug
lunny May 16, 2024
4fc10e8
Merge branch 'main' into lunny/azure_blob_storage
lunny May 16, 2024
755f13a
Add documentations
lunny May 16, 2024
37c2ffc
Add azureblob test for mssql tests
lunny May 16, 2024
1d0d1ed
Fix ci
lunny May 16, 2024
8dcf589
Fix ci
lunny May 16, 2024
c3f1090
Fix read size
lunny May 17, 2024
9741953
Merge branch 'main' into lunny/azure_blob_storage
lunny May 17, 2024
22a247a
Fix test bug
lunny May 17, 2024
929a4e3
Fix bug
lunny May 17, 2024
0f32f6b
Fix servedirect
lunny May 17, 2024
8a01693
Remove comment code
lunny May 17, 2024
068bb24
Fix test
lunny May 17, 2024
15c439d
Fix removing non-exist attachment
lunny May 18, 2024
50bf919
improve test
lunny May 19, 2024
ca6a1dc
package version should not have space
lunny May 20, 2024
c23e531
Fix the problem that azurite's special check for SASURL
lunny May 21, 2024
4b88ab4
Fix test
lunny May 21, 2024
baca070
Merge branch 'main' into lunny/azure_blob_storage
lunny May 21, 2024
4511c13
Revert unnecessary change
lunny May 22, 2024
3646822
Merge branch 'main' into lunny/azure_blob_storage
lunny May 22, 2024
a3c56dd
Apply suggestions from code review
lunny May 23, 2024
e76bf65
Update documentations
lunny May 23, 2024
391fd1e
Fix bug
lunny May 27, 2024
8fb3532
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 29, 2024
6c35486
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 29, 2024
a28b05e
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 29, 2024
066f05f
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 29, 2024
f2ae422
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 29, 2024
55de09c
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 30, 2024
1ef76dc
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 30, 2024
f81c683
Merge branch 'main' into lunny/azure_blob_storage
GiteaBot May 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix bug
  • Loading branch information
lunny committed May 16, 2024
commit de67cea09204ee9a9163e872cfdc80eaa0f6845c
88 changes: 37 additions & 51 deletions modules/storage/azureblob.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,65 +30,58 @@ import (
var _ Object = &azureBlobObject{}

type azureBlobObject struct {
BlobClient *blob.Client
Context *context.Context
blobClient *blob.Client
Context context.Context
Name string
Size int64
ModTime *time.Time

Offset int64
offset int64
}

func (a *azureBlobObject) downloadBuffer(p []byte) (int, error) {
if a.Offset > a.Size {
func (a *azureBlobObject) Read(p []byte) (int, error) {
// TODO: improve the performance, we can implement another interface, maybe implement io.WriteTo
if a.offset >= a.Size {
return 0, io.EOF
}
count := a.Size - a.Offset
pl := int64(len(p))
if pl > count {
p = p[0:count]
} else {
count = pl
}
res, err := a.BlobClient.DownloadBuffer(*a.Context, p, &blob.DownloadBufferOptions{
count := min(int64(len(p)), a.Size-a.offset)

res, err := a.blobClient.DownloadBuffer(a.Context, p, &blob.DownloadBufferOptions{
Range: blob.HTTPRange{
Offset: a.Offset,
Offset: a.offset,
Count: count,
},
})
if err != nil {
return 0, convertAzureBlobErr(err)
}
a.Offset + 10000 = count
a.offset += res

return int(res), nil
}

func (a *azureBlobObject) Close() error {
a.offset = 0
return nil
}

func (a *azureBlobObject) Read(p []byte) (int, error) {
c, err := a.downloadBuffer(p)
return c, err
}

func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
switch whence {
default:
return 0, errors.New("Seek: invalid whence")
case io.SeekStart:
offset += 0
a.offset = offset
case io.SeekCurrent:
offset += a.Offset
a.offset += offset
case io.SeekEnd:
offset += a.Size
a.offset = a.Size - offset
default:
return 0, errors.New("Seek: invalid whence")
}
if offset < 0 {

if a.offset > a.Size {
return 0, errors.New("Seek: invalid offset")
} else if a.offset < 0 {
return 0, errors.New("Seek: invalid offset")
}
a.Offset = offset
return offset, nil
return a.offset, nil
}

func (a *azureBlobObject) Stat() (os.FileInfo, error) {
Expand All @@ -103,11 +96,10 @@ var _ ObjectStorage = &AzureBlobStorage{}

// AzureStorage returns a azure blob storage
type AzureBlobStorage struct {
cfg *setting.AzureBlobStorageConfig
ctx context.Context
credential *azblob.SharedKeyCredential
client *azblob.Client
containerClient *container.Client
cfg *setting.AzureBlobStorageConfig
ctx context.Context
credential *azblob.SharedKeyCredential
client *azblob.Client
}

func convertAzureBlobErr(err error) error {
Expand Down Expand Up @@ -135,7 +127,7 @@ func NewAzureBlobStorage(ctx context.Context, cfg *setting.Storage) (ObjectStora
if err != nil {
return nil, convertAzureBlobErr(err)
}
client, err := azblob.NewClientWithSharedKeyCredential(config.Endpoint+"/"+config.AccountName, cred, &azblob.ClientOptions{})
client, err := azblob.NewClientWithSharedKeyCredential(config.Endpoint, cred, &azblob.ClientOptions{})
if err != nil {
return nil, convertAzureBlobErr(err)
}
Expand All @@ -149,11 +141,10 @@ func NewAzureBlobStorage(ctx context.Context, cfg *setting.Storage) (ObjectStora
}

return &AzureBlobStorage{
cfg: &config,
ctx: ctx,
credential: cred,
client: client,
containerClient: client.ServiceClient().NewContainerClient(config.Container),
cfg: &config,
ctx: ctx,
credential: cred,
client: client,
}, nil
}

Expand Down Expand Up @@ -181,8 +172,8 @@ func (a *AzureBlobStorage) Open(path string) (Object, error) {
return nil, convertAzureBlobErr(err)
}
return &azureBlobObject{
Context: &a.ctx,
BlobClient: blobClient,
Context: a.ctx,
blobClient: blobClient,
Name: a.getObjectNameFromPath(path),
Size: *res.ContentLength,
ModTime: res.LastModified,
Expand All @@ -191,11 +182,7 @@ func (a *AzureBlobStorage) Open(path string) (Object, error) {

// Save saves a file to azure blob storage
func (a *AzureBlobStorage) Save(path string, r io.Reader, size int64) (int64, error) {
client, err := azblob.NewClientFromConnectionString(a.cfg.ConnectionString(), &azblob.ClientOptions{})
if err != nil {
return 0, convertAzureBlobErr(err)
}
_, err = client.UploadStream(
_, err := a.client.UploadStream(
a.ctx,
a.cfg.Container,
a.buildAzureBlobPath(path),
Expand Down Expand Up @@ -312,8 +299,8 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o
return convertAzureBlobErr(err)
}
object := &azureBlobObject{
Context: &a.ctx,
BlobClient: blobClient,
Context: a.ctx,
blobClient: blobClient,
Name: *object.Name,
Size: *object.Properties.ContentLength,
ModTime: object.Properties.LastModified,
Expand All @@ -331,8 +318,7 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o

// Delete delete a file
func (a *AzureBlobStorage) getBlobClient(path string) (*blob.Client, error) {
// blob.NewClient(blobURL string, cred azcore.TokenCredential, options *blob.ClientOptions)
return blob.NewClientFromConnectionString(a.cfg.ConnectionString(), a.cfg.Container, a.buildAzureBlobPath(path), &blob.ClientOptions{})
return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path)), nil
}

func init() {
Expand Down
0