Recreate volumes on volume configuration change

Signed-off-by: Joana Hrotko <[email protected]>

cmd/compose/create.go

@@ -32,20 +32,21 @@ import (
 )
 
 type createOptions struct {
-	Build         bool
-	noBuild       bool
-	Pull          string
-	pullChanged   bool
-	removeOrphans bool
-	ignoreOrphans bool
-	forceRecreate bool
-	noRecreate    bool
-	recreateDeps  bool
-	noInherit     bool
-	timeChanged   bool
-	timeout       int
-	quietPull     bool
-	scale         []string
+	Build           bool
+	noBuild         bool
+	Pull            string
+	pullChanged     bool
+	removeOrphans   bool
+	ignoreOrphans   bool
+	forceRecreate   bool
+	noRecreate      bool
+	recreateDeps    bool
+	noInherit       bool
+	timeChanged     bool
+	timeout         int
+	quietPull       bool
+	scale           []string
+	recreateVolumes bool
 }
 
 func createCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command {

cmd/compose/up.go

@@ -148,6 +148,7 @@ func upCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *c
 	flags.BoolVar(&create.noBuild, "no-build", false, "Don't build an image, even if it's policy")
 	flags.StringVar(&create.Pull, "pull", "policy", `Pull image before running ("always"|"missing"|"never")`)
 	flags.BoolVar(&create.removeOrphans, "remove-orphans", false, "Remove containers for services not defined in the Compose file")
+	flags.BoolVar(&create.recreateVolumes, "recreate-volumes", false, "Recreate volumes when volume configuration in the Compose file changes.")
 	flags.StringArrayVar(&create.scale, "scale", []string{}, "Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.")
 	flags.BoolVar(&up.noColor, "no-color", false, "Produce monochrome output")
 	flags.BoolVar(&up.noPrefix, "no-log-prefix", false, "Don't print prefix in logs")
@@ -255,6 +256,7 @@ func runUp(
 		Inherit:              !createOptions.noInherit,
 		Timeout:              createOptions.GetTimeout(),
 		QuietPull:            createOptions.quietPull,
+		RecreateVolumes:      createOptions.recreateVolumes,
 	}
 
 	if upOptions.noStart {

docs/reference/compose_up.md

@@ -45,6 +45,7 @@ If the process is interrupted using `SIGINT` (ctrl + C) or `SIGTERM`, the contai
 | `--no-start`                   | `bool`        |          | Don't start the services after creating them                                                                                                        |
 | `--pull`                       | `string`      | `policy` | Pull image before running ("always"\|"missing"\|"never")                                                                                            |
 | `--quiet-pull`                 | `bool`        |          | Pull without printing progress information                                                                                                          |
+| `--recreate-volumes`           | `bool`        |          | Recreate volumes when volume configuration in the Compose file changes.                                                                             |
 | `--remove-orphans`             | `bool`        |          | Remove containers for services not defined in the Compose file                                                                                      |
 | `-V`, `--renew-anon-volumes`   | `bool`        |          | Recreate anonymous volumes instead of retrieving data from the previous containers                                                                  |
 | `--scale`                      | `stringArray` |          | Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.                                                       |

docs/reference/docker_compose_up.yaml

@@ -221,6 +221,17 @@ options:
       experimentalcli: false
       kubernetes: false
       swarm: false
+    - option: recreate-volumes
+      value_type: bool
+      default_value: "false"
+      description: |
+        Recreate volumes when volume configuration in the Compose file changes.
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
     - option: remove-orphans
       value_type: bool
       default_value: "false"

pkg/api/api.go

@@ -202,6 +202,8 @@ type CreateOptions struct {
 	Timeout *time.Duration
 	// QuietPull makes the pulling process quiet
 	QuietPull bool
+	// Allow recreate volumes when volumes configuration changes
+	RecreateVolumes bool
 }
 
 // StartOptions group options of the Start API

pkg/api/labels.go

@@ -31,6 +31,8 @@ const (
 	ServiceLabel = "com.docker.compose.service"
 	// ConfigHashLabel stores configuration hash for a compose service
 	ConfigHashLabel = "com.docker.compose.config-hash"
+	// VolumeConfigHashLabel stores configuration hash for a compose volume
+	VolumeConfigHashLabel = "com.docker.compose.volume-config-hash"
 	// ContainerNumberLabel stores the container index of a replicated service
 	ContainerNumberLabel = "com.docker.compose.container-number"
 	// VolumeLabel allow to track resource related to a compose volume

pkg/compose/create.go

@@ -65,6 +65,8 @@ type createConfigs struct {
 	Links     []string
 }
 
+type ComposeVolumes map[string]*volumetypes.Volume
+
 func (s *composeService) Create(ctx context.Context, project *types.Project, createOpts api.CreateOptions) error {
 	return progress.RunWithTitle(ctx, func(ctx context.Context) error {
 		return s.create(ctx, project, createOpts)
@@ -81,12 +83,6 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt
 		return err
 	}
 
-	var observedState Containers
-	observedState, err = s.getContainers(ctx, project.Name, oneOffInclude, true)
-	if err != nil {
-		return err
-	}
-
 	err = s.ensureImagesExists(ctx, project, options.Build, options.QuietPull)
 	if err != nil {
 		return err
@@ -98,10 +94,15 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt
 		return err
 	}
 
-	if err := s.ensureProjectVolumes(ctx, project); err != nil {
+	if err := s.ensureProjectVolumes(ctx, project, options.RecreateVolumes); err != nil {
 		return err
 	}
 
+	var observedState Containers
+	observedState, err = s.getContainers(ctx, project.Name, oneOffInclude, true)
+	if err != nil {
+		return err
+	}
 	orphans := observedState.filter(isOrphaned(project))
 	if len(orphans) > 0 && !options.IgnoreOrphans {
 		if options.RemoveOrphans {
@@ -119,6 +120,25 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt
 	return newConvergence(options.Services, observedState, s).apply(ctx, project, options)
 }
 
+func (s *composeService) getVolumes(ctx context.Context, projectName string) (ComposeVolumes, error) {
+	volumeList, err := s.apiClient().VolumeList(ctx, volumetypes.ListOptions{
+		Filters: filters.NewArgs(projectFilter(projectName)),
+	})
+	if err != nil {
+		return nil, err
+	}
+	if volumeList.Volumes == nil {
+		return nil, nil
+	}
+
+	volumes := make(ComposeVolumes, len(volumeList.Volumes))
+	for _, v := range volumeList.Volumes {
+		name := v.Labels[api.VolumeLabel]
+		volumes[name] = v
+	}
+	return volumes, nil
+}
+
 func prepareNetworks(project *types.Project) {
 	for k, nw := range project.Networks {
 		nw.Labels = nw.Labels.Add(api.NetworkLabel, k)
@@ -139,71 +159,151 @@ func (s *composeService) ensureNetworks(ctx context.Context, networks types.Netw
 	return nil
 }
 
-func (s *composeService) ensureProjectVolumes(ctx context.Context, project *types.Project) error {
+func (s *composeService) ensureProjectVolumes(ctx context.Context, project *types.Project, recreateVolumes bool) error {
+	// create observed state for volumes
+	observedVolumesState, err := s.getVolumes(ctx, project.Name)
+	if err != nil {
+		return err
+	}
 	for k, volume := range project.Volumes {
 		volume.Labels = volume.Labels.Add(api.VolumeLabel, k)
 		volume.Labels = volume.Labels.Add(api.ProjectLabel, project.Name)
 		volume.Labels = volume.Labels.Add(api.VersionLabel, api.ComposeVersion)
-		err := s.ensureVolume(ctx, volume, project.Name)
+		volumeConfigHash, err := VolumeHash(volume)
+		if err != nil {
+			return err
+		}
+		volume.Labels = volume.Labels.Add(api.VolumeConfigHashLabel, volumeConfigHash)
+
+		err = s.ensureVolume(ctx, volume, project.Name)
+		if err != nil {
+			return err
+		}
+
+		actualVolume, exists := observedVolumesState[k]
+		if !exists {
+			continue
+		}
+		// Check if it's necessary to recreate the volume by comparing
+		// with the current volume config hash
+		shouldRecreateVolume, err := shouldUpdateVolume(volume, actualVolume)
 		if err != nil {
 			return err
 		}
+		if !shouldRecreateVolume {
+			continue
+		}
+		if !recreateVolumes && shouldRecreateVolume {
+			logrus.Warnf("Warning: An update has been made to the volume %s. To apply this update to your running Compose project,"+
+				"please use the up command with the --recreate-volume option. Note that recreating the volume will remove the existing "+
+				"volume in order to update it.", k)
+			continue
+		}
+		if recreateVolumes && shouldRecreateVolume {
+			err = s.recreateVolume(ctx, project, k, volume)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
-	err := func() error {
-		if s.manageDesktopFileSharesEnabled(ctx) {
-			// collect all the bind mount paths and try to set up file shares in
-			// Docker Desktop for them
-			var paths []string
-			for _, svcName := range project.ServiceNames() {
-				svc := project.Services[svcName]
-				for _, vol := range svc.Volumes {
-					if vol.Type != string(mount.TypeBind) {
-						continue
-					}
-					p := filepath.Clean(vol.Source)
-					if !filepath.IsAbs(p) {
-						return fmt.Errorf("file share path is not absolute: %s", p)
-					}
-					if fi, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) {
-						// actual directory will be implicitly created when the
-						// file share is initialized if it doesn't exist, so
-						// need to filter out any that should not be auto-created
-						if vol.Bind != nil && !vol.Bind.CreateHostPath {
-							logrus.Debugf("Skipping creating file share for %q: does not exist and `create_host_path` is false", p)
-							continue
-						}
-					} else if err != nil {
-						// if we can't read the path, we won't be able to make
-						// a file share for it
-						logrus.Debugf("Skipping creating file share for %q: %v", p, err)
-						continue
-					} else if !fi.IsDir() {
-						// ignore files & special types (e.g. Unix sockets)
-						logrus.Debugf("Skipping creating file share for %q: not a directory", p)
+	err = s.configureFileShares(ctx, project)
+	if err != nil {
+		progress.ContextWriter(ctx).TailMsgf("Failed to prepare Synchronized file shares: %v", err)
+	}
+	return nil
+}
+
+func (s *composeService) configureFileShares(ctx context.Context, project *types.Project) error {
+	if s.manageDesktopFileSharesEnabled(ctx) {
+		// collect all the bind mount paths and try to set up file shares in
+		// Docker Desktop for them
+		var paths []string
+		for _, svcName := range project.ServiceNames() {
+			svc := project.Services[svcName]
+			for _, vol := range svc.Volumes {
+				if vol.Type != string(mount.TypeBind) {
+					continue
+				}
+				p := filepath.Clean(vol.Source)
+				if !filepath.IsAbs(p) {
+					return fmt.Errorf("file share path is not absolute: %s", p)
+				}
+				if fi, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) {
+					// actual directory will be implicitly created when the
+					// file share is initialized if it doesn't exist, so
+					// need to filter out any that should not be auto-created
+					if vol.Bind != nil && !vol.Bind.CreateHostPath {
+						logrus.Debugf("Skipping creating file share for %q: does not exist and `create_host_path` is false", p)
 						continue
 					}
-
-					paths = append(paths, p)
+				} else if err != nil {
+					// if we can't read the path, we won't be able to make
+					// a file share for it
+					logrus.Debugf("Skipping creating file share for %q: %v", p, err)
+					continue
+				} else if !fi.IsDir() {
+					// ignore files & special types (e.g. Unix sockets)
+					logrus.Debugf("Skipping creating file share for %q: not a directory", p)
+					continue
 				}
+
+				paths = append(paths, p)
 			}
+		}
+
+		// remove duplicate/unnecessary child paths and sort them for predictability
+		paths = pathutil.EncompassingPaths(paths)
+		sort.Strings(paths)
 
-			// remove duplicate/unnecessary child paths and sort them for predictability
-			paths = pathutil.EncompassingPaths(paths)
-			sort.Strings(paths)
+		fileShareManager := desktop.NewFileShareManager(s.desktopCli, project.Name, paths)
+		if err := fileShareManager.EnsureExists(ctx); err != nil {
+			return fmt.Errorf("initializing file shares: %w", err)
+		}
+	}
+	return nil
+}
 
-			fileShareManager := desktop.NewFileShareManager(s.desktopCli, project.Name, paths)
-			if err := fileShareManager.EnsureExists(ctx); err != nil {
-				return fmt.Errorf("initializing file shares: %w", err)
+func (s *composeService) recreateVolume(ctx context.Context, project *types.Project, volumeID string, volumeConfig types.VolumeConfig) error {
+	// need to recreate containers associated with the volume first
+	selectedService := []string{}
+	for _, s := range project.Services {
+		for _, v := range s.Volumes {
+			if v.Source == volumeID {
+				selectedService = append(selectedService, s.Name)
 			}
 		}
-		return nil
-	}()
+	}
+	err := s.Remove(ctx, project.Name, api.RemoveOptions{
+		Project:  project,
+		Services: selectedService,
+		Force:    true,
+		Stop:     true,
+	})
+	if err != nil {
+		return err
+	}
+	// remove volume to update it
+	w := progress.ContextWriter(ctx)
+	err = s.removeVolume(ctx, volumeConfig.Name, w)
+	if err != nil {
+		return err
+	}
+
+	// create the volume
+	return s.createVolume(ctx, volumeConfig)
+}
 
+func shouldUpdateVolume(config types.VolumeConfig, volume *volumetypes.Volume) (bool, error) {
+	previousHash, ok := volume.Labels[api.VolumeConfigHashLabel]
+	if !ok {
+		return false, nil
+	}
+	currentHash, err := VolumeHash(config)
 	if err != nil {
-		progress.ContextWriter(ctx).TailMsgf("Failed to prepare Synchronized file shares: %v", err)
+		return false, fmt.Errorf("could not get hash label for volume config %s", config.Name)
 	}
-	return nil
+	return previousHash != currentHash, nil
 }
 
 func (s *composeService) getCreateConfigs(ctx context.Context,