8000 Extend driver.ListLayers() · containers/storage@972f7c7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 972f7c7

Browse files
nalindnalind
committed
Extend driver.ListLayers()
Implement ListLayers() for the aufs, btrfs, devicemapper, and vfs drivers, along with a unit test for them. Have layerStore.Wipe() try to remove remaining listed layers after it removes the layers that the layerStore knew of. Close() a dangling ReadCloser in NaiveCreateFromTemplate. Switch from using plain defer to using t.Cleanup() to handle deleting layers that tests create, have the addManyLayers() test function do so as well, and have unit tests retry layer removal up to a surprisingly large number of times on EBUSY. Remove vfs.CopyDir, which near as I can tell isn't referenced anywhere. Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
1 parent 6e387ad commit 972f7c7

19 files changed

+249
-107
lines changed

‎drivers/aufs/aufs.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,9 +251,22 @@ func (a *Driver) Exists(id string) bool {
251251
return true
252252
}
253253

254-
// List layers (not including additional image stores)
254+
// ListLayers(), when implemented, is used when checking for errors. It shoud return
255+
// all of the layers known to the driver.
255256
func (a *Driver) ListLayers() ([]string, error) {
256-
return nil, graphdriver.ErrNotSupported
257+
diffsDir := filepath.Join(a.rootPath(), "diff")
258+
entries, err := os.ReadDir(diffsDir)
259+
if err != nil {
260+
return nil, err
261+
}
262+
results := make([]string, 0, len(entries))
263+
for _, entry := range entries {
264+
if !entry.IsDir() {
265+
continue
266+
}
267+
results = append(results, entry.Name())
268+
}
269+
return results, nil
257270
}
258271

259272
// AdditionalImageStores returns additional image stores supported by the driver

‎drivers/aufs/aufs_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -824,3 +824,7 @@ func TestAufsCreateFromTemplate(t *testing.T) {
824824
func TestAufsEcho(t *testing.T) {
825825
graphtest.DriverTestEcho(t, "aufs")
826826
}
827+
828+
func TestAufsListLayers(t *testing.T) {
829+
graphtest.DriverTestListLayers(t, "aufs")
830+
}

‎drivers/btrfs/btrfs.go‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
7070
if err != nil {
7171
return nil, err
7272
}
73-
if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil {
73+
if err := idtools.MkdirAllAs(filepath.Join(home, "subvolumes"), 0700, rootUID, rootGID); err != nil {
7474
return nil, err
7575
}
7676

@@ -89,14 +89,16 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
8989
gidMaps: options.GIDMaps,
9090
options: opt,
9191
}
92+
driver.LayerIDMapUpdater = graphdriver.NewNaiveLayerIDMapUpdater(driver)
93+
driver.DiffDriver = graphdriver.NewNaiveDiffDriver(driver, driver.LayerIDMapUpdater)
9294

9395
if userDiskQuota {
9496
if err := driver.enableQuota(); err != nil {
9597
return nil, err
9698
}
9799
}
98100

99-
return graphdriver.NewNaiveDiffDriver(driver, graphdriver.NewNaiveLayerIDMapUpdater(driver)), nil
101+
return driver, nil
100102
}
101103

102104
func parseOptions(opt []string) (btrfsOptions, bool, error) {
@@ -119,7 +121,7 @@ func parseOptions(opt []string) (btrfsOptions, bool, error) {
119121
case "btrfs.mountopt":
120122
return options, userDiskQuota, fmt.Errorf("btrfs driver does not support mount options")
121123
default:
122-
return options, userDiskQuota, fmt.Errorf("unknown option %s", key)
124+
return options, userDiskQuota, fmt.Errorf("unknown option %s (%q)", key, option)
123125
}
124126
}
125127
return options, userDiskQuota, nil
@@ -134,6 +136,8 @@ type Driver struct {
134136
options btrfsOptions
135137
quotaEnabled bool
136138
once sync.Once
139+
graphdriver.LayerIDMapUpdater
140+
graphdriver.DiffDriver
137141
}
138142

139143
// String prints the name of the driver (btrfs).
@@ -479,8 +483,8 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts
479483

480484
// Create the filesystem with given id.
481485
func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
482-
quotas := path.Join(d.home, "quotas")
483-
subvolumes := path.Join(d.home, "subvolumes")
486+
quotas := d.quotasDir()
487+
subvolumes := d.subvolumesDir()
484488
rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
485489
if err != nil {
486490
return err
@@ -560,7 +564,7 @@ func (d *Driver) parseStorageOpt(storageOpt map[string]string, driver *Driver) e
560564
}
561565
driver.options.size = uint64(size)
562566
default:
563-
return fmt.Errorf("unknown option %s", key)
567+
return fmt.Errorf("unknown option %s (%q)", key, storageOpt)
564568
}
565569
}
566570

@@ -679,9 +683,22 @@ func (d *Driver) Exists(id string) bool {
679683
return err == nil
680684
}
681685

682-
// List layers (not including additional image stores)
686+
// ListLayers(), when implemented, is used when checking for errors. It should return
687+
// all of the layers known to the driver.
683688
func (d *Driver) ListLayers() ([]string, error) {
684-
return nil, graphdriver.ErrNotSupported
689+
subvolumesDir := filepath.Join(d.home, "subvolumes")
690+
entries, err := os.ReadDir(subvolumesDir)
691+
if err != nil {
692+
return nil, err
693+
}
694+
results := make([]string, 0, len(entries))
695+
for _, entry := range entries {
696+
if !entry.IsDir() {
697+
continue
698+
}
699+
results = append(results, entry.Name())
700+
}
701+
return results, nil
685702
}
686703

687704
// AdditionalImageStores returns additional image stores supported by the driver

‎drivers/btrfs/btrfs_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ func TestBtrfsEcho(t *testing.T) {
6868
graphtest.DriverTestEcho(t, "btrfs")
6969
}
7070

71+
func TestBtrfsListLayers(t *testing.T) {
72+
graphtest.DriverTestListLayers(t, "btrfs")
73+
}
74+
7175
func TestBtrfsTeardown(t *testing.T) {
7276
graphtest.PutDriver(t)
7377
}

‎drivers/devmapper/deviceset.go‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,6 +1001,10 @@ func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error {
10011001
devices.Lock()
10021002
defer devices.Unlock()
10031003

1004+
if devices.filesystem == "" {
1005+
devices.filesystem = determineDefaultFS()
1006+
}
1007+
10041008
if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
10051009
return err
10061010
}
@@ -1707,6 +1711,9 @@ func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
17071711
if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil {
17081712
return err
17091713
}
1714+
if err := idtools.MkdirAs(filepath.Join(devices.root, "mnt"), 0700, uid, gid); err != nil && !errors.Is(err, os.ErrExist) {
1715+
return err
1716+
}
17101717

17111718
prevSetupConfig, err := readLVMConfig(devices.root)
17121719
if err != nil {
@@ -2317,7 +2324,7 @@ func (devices *DeviceSet) Shutdown(home string) error {
23172324
info.lock.Lock()
23182325
devices.Lock()
23192326
if err := devices.deactivateDevice(info); err != nil {
2320-
logrus.Debugf("devmapper: Shutdown deactivate base , error: %s", err)
2327+
logrus.Debugf("devmapper: Shutdown deactivate base, error: %s", err)
23212328
}
23222329
devices.Unlock()
23232330
info.lock.Unlock()
@@ -2326,7 +2333,7 @@ func (devices *DeviceSet) Shutdown(home string) error {
23262333
devices.Lock()
23272334
if devices.thinPoolDevice == "" {
23282335
if err := devices.deactivatePool(); err != nil {
2329-
logrus.Debugf("devmapper: Shutdown deactivate pool , error: %s", err)
2336+
logrus.Debugf("devmapper: Shutdown deactivate pool, error: %s", err)
23302337
}
23312338
}
23322339
devices.Unlock()
@@ -2483,6 +2490,21 @@ func (devices *DeviceSet) List() []string {
24832490
return ids
24842491
}
24852492

2493+
// ListLayers returns a list of device IDs, omitting the ""/"base" device.
2494+
func (devices *DeviceSet) ListLayers() ([]string, error) {
2495+
devices.Lock()
2496+
defer devices.Unlock()
2497+
249 D742 8+
ids := make([]string, 0, len(devices.Devices))
2499+
for k := range devices.Devices {
2500+
if k == "" {
2501+
continue
2502+
}
2503+
ids = append(ids, k)
2504+
}
2505+
return ids, nil
2506+
}
2507+
24862508
func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
24872509
var params string
24882510
_, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)

‎drivers/devmapper/devmapper_test.go‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,18 @@ func TestDevmapperCreateFromTemplate(t *testing.T) {
9191
graphtest.DriverTestCreateFromTemplate(t, "devicemapper", "test=1")
9292
}
9393

94-
func TestDevmapperTeardown(t *testing.T) {
95-
graphtest.PutDriver(t)
96-
}
97-
9894
func TestDevmapperEcho(t *testing.T) {
9995
graphtest.DriverTestEcho(t, "devicemapper", "test=1")
10096
}
10197

98+
func TestDevmapperListLayers(t *testing.T) {
99+
graphtest.DriverTestListLayers(t, "devicemapper", "test=1")
100+
}
101+
102+
func TestDevmapperTeardown(t *testing.T) {
103+
graphtest.PutDriver(t)
104+
}
105+
102106
func TestDevmapperReduceLoopBackSize(t *testing.T) {
103107
tenMB := int64(10 * 1024 * 1024)
104108
testChangeLoopBackSize(t, -tenMB, defaultDataLoopbackSize, defaultMetaDataLoopbackSize)
@@ -110,7 +114,7 @@ func TestDevmapperIncreaseLoopBackSize(t *testing.T) {
110114
}
111115

112116
func testChangeLoopBackSize(t *testing.T, delta, expectDataSize, expectMetaDataSize int64) {
113-
driver := graphtest.GetDriver(t, "devicemapper", "test=1").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
117+
driver := graphtest.GetDriver(t, "devicemapper" D742 , "test=1").(*graphtest.Driver).Driver.(*Driver).DiffDriver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
114118
defer graphtest.PutDriver(t)
115119
// make sure data or metadata loopback size are the default size
116120
if s := driver.DeviceSet.Status(); s.Data.Total != uint64(defaultDataLoopbackSize) || s.Metadata.Total != uint64(defaultMetaDataLoopbackSize) {
@@ -128,9 +132,9 @@ func testChangeLoopBackSize(t *testing.T, delta, expectDataSize, expectMetaDataS
128132
if err != nil {
129133
t.Fatalf("error creating devicemapper driver: %v", err)
130134
}
131-
driver = d.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
135+
driver = d.(*Driver).DiffDriver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
132136
if s := driver.DeviceSet.Status(); s.Data.Total != uint64(expectDataSize) || s.Metadata.Total != uint64(expectMetaDataSize) {
133-
t.Fatal("data or metadata loop back size is incorrect")
137+
t.Fatalf("data or metadata loop back size is incorrect (data actual/expected=%d/%d, metadata actual/expected = %d/%d)", s.Data.Total, expectDataSize, s.Metadata.Total, expectMetaDataSize)
134138
}
135139
if err := driver.Cleanup(); err != nil {
136140
t.Fatal(err)
@@ -139,7 +143,7 @@ func testChangeLoopBackSize(t *testing.T, delta, expectDataSize, expectMetaDataS
139143

140144
// Make sure devices.Lock() has been release upon return from cleanupDeletedDevices() function
141145
func TestDevmapperLockReleasedDeviceDeletion(t *testing.T) {
142-
driver := graphtest.GetDriver(t, "devicemapper", "test=1").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
146+
driver := graphtest.GetDriver(t, "devicemapper", "test=1").(*graphtest.Driver).Driver.(*Driver).DiffDriver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
143147
defer graphtest.PutDriver(t)
144148

145149
// Call cleanupDeletedDevices() and after the call take and release
@@ -173,7 +177,7 @@ func TestDevmapperMountLeaks(t *testing.T) {
173177
t.Skipf("kernel version <3.18.0 and so is missing torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe.")
174178
}
175179

176-
driver := graphtest.GetDriver(t, "devicemapper", "test=1", "dm.use_deferred_removal=false", "dm.use_deferred_deletion=false").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
180+
driver := graphtest.GetDriver(t, "devicemapper", "test=1", "dm.use_deferred_removal=false", "dm.use_deferred_deletion=false").(*graphtest.Driver).Driver.(*Driver).DiffDriver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
177181
defer graphtest.PutDriver(t)
178182

179183
// We need to create a new (dummy) device.

‎drivers/devmapper/driver.go‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type Driver struct {
3434
gidMaps []idtools.IDMap
3535
ctr *graphdriver.RefCounter
3636
locker *locker.Locker
37+
graphdriver.DiffDriver
38+
graphdriver.LayerIDMapUpdater
3739
}
3840

3941
// Init creates a driver with the given home and the set of options.
@@ -55,8 +57,9 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
5557
ctr: graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
5658
locker: locker.New(),
5759
}
58-
59-
return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil
60+
d.LayerIDMapUpdater = graphdriver.NewNaiveLayerIDMapUpdater(d)
61+
d.DiffDriver = graphdriver.NewNaiveDiffDriver(d, d.LayerIDMapUpdater)
62+
return d, nil
6063
}
6164

6265
func (d *Driver) String() string {
@@ -267,11 +270,6 @@ func (d *Driver) Exists(id string) bool {
267270
return d.DeviceSet.HasDevice(id)
268271
}
269272

270-
// List layers (not including additional image stores)
271-
func (d *Driver) ListLayers() ([]string, error) {
272-
return nil, graphdriver.ErrNotSupported
273-
}
274-
275273
// AdditionalImageStores returns additional image stores supported by the driver
276274
func (d *Driver) AdditionalImageStores() []string {
277275
return nil

‎drivers/graphtest/graphbench_unix.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func DriverBenchDeepLayerDiff(b *testing.B, layerCount int, drivername string, d
198198
b.Fatal(err)
199199
}
200200

201-
topLayer, err := addManyLayers(driver, base, layerCount)
201+
topLayer, err := addManyLayers(b, driver, base, layerCount)
202202
if err != nil {
203203
b.Fatal(err)
204204
}
@@ -232,7 +232,7 @@ func DriverBenchDeepLayerRead(b *testing.B, layerCount int, drivername string, d
232232
b.Fatal(err)
233233
}
234234

235-
topLayer, err := addManyLayers(driver, base, layerCount)
235+
topLayer, err := addManyLayers(b, driver, base, layerCount)
236236
if err != nil {
237237
b.Fatal(err)
238238
}

0 commit comments

Comments
 (0)
0