diff --git a/cmd/compose/create.go b/cmd/compose/create.go index 1fe05e98dc4..d14b1cc36cf 100644 --- a/cmd/compose/create.go +++ b/cmd/compose/create.go @@ -32,20 +32,21 @@ import ( ) type createOptions struct { - Build bool - noBuild bool - Pull string - pullChanged bool - removeOrphans bool - ignoreOrphans bool - forceRecreate bool - noRecreate bool - recreateDeps bool - noInherit bool - timeChanged bool - timeout int - quietPull bool - scale []string + Build bool + noBuild bool + Pull string + pullChanged bool + removeOrphans bool + ignoreOrphans bool + forceRecreate bool + noRecreate bool + recreateDeps bool + noInherit bool + timeChanged bool + timeout int + quietPull bool + scale []string + recreateVolumes bool } func createCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command { diff --git a/cmd/compose/up.go b/cmd/compose/up.go index c7cd3ef76b0..6208cbd500e 100644 --- a/cmd/compose/up.go +++ b/cmd/compose/up.go @@ -148,6 +148,7 @@ func upCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *c flags.BoolVar(&create.noBuild, "no-build", false, "Don't build an image, even if it's policy") flags.StringVar(&create.Pull, "pull", "policy", `Pull image before running ("always"|"missing"|"never")`) flags.BoolVar(&create.removeOrphans, "remove-orphans", false, "Remove containers for services not defined in the Compose file") + flags.BoolVar(&create.recreateVolumes, "recreate-volumes", false, "Recreate volumes when volume configuration in the Compose file changes.") flags.StringArrayVar(&create.scale, "scale", []string{}, "Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.") flags.BoolVar(&up.noColor, "no-color", false, "Produce monochrome output") flags.BoolVar(&up.noPrefix, "no-log-prefix", false, "Don't print prefix in logs") @@ -255,6 +256,7 @@ func runUp( Inherit: !createOptions.noInherit, Timeout: createOptions.GetTimeout(), QuietPull: createOptions.quietPull, + RecreateVolumes: createOptions.recreateVolumes, } if upOptions.noStart { diff --git a/docs/reference/compose_up.md b/docs/reference/compose_up.md index 295000734b1..ebe09435b87 100644 --- a/docs/reference/compose_up.md +++ b/docs/reference/compose_up.md @@ -45,6 +45,7 @@ If the process is interrupted using `SIGINT` (ctrl + C) or `SIGTERM`, the contai | `--no-start` | `bool` | | Don't start the services after creating them | | `--pull` | `string` | `policy` | Pull image before running ("always"\|"missing"\|"never") | | `--quiet-pull` | `bool` | | Pull without printing progress information | +| `--recreate-volumes` | `bool` | | Recreate volumes when volume configuration in the Compose file changes. | | `--remove-orphans` | `bool` | | Remove containers for services not defined in the Compose file | | `-V`, `--renew-anon-volumes` | `bool` | | Recreate anonymous volumes instead of retrieving data from the previous containers | | `--scale` | `stringArray` | | Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present. | diff --git a/docs/reference/docker_compose_up.yaml b/docs/reference/docker_compose_up.yaml index 4c895f6a65b..d00046e9aa0 100644 --- a/docs/reference/docker_compose_up.yaml +++ b/docs/reference/docker_compose_up.yaml @@ -221,6 +221,17 @@ options: experimentalcli: false kubernetes: false swarm: false + - option: recreate-volumes + value_type: bool + default_value: "false" + description: | + Recreate volumes when volume configuration in the Compose file changes. + deprecated: false + hidden: false + experimental: false + experimentalcli: false + kubernetes: false + swarm: false - option: remove-orphans value_type: bool default_value: "false" diff --git a/pkg/api/api.go b/pkg/api/api.go index e48cde46d39..f2e90ee14fa 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -202,6 +202,8 @@ type CreateOptions struct { Timeout *time.Duration // QuietPull makes the pulling process quiet QuietPull bool + // Allow recreate volumes when volumes configuration changes + RecreateVolumes bool } // StartOptions group options of the Start API diff --git a/pkg/api/labels.go b/pkg/api/labels.go index ba110c83147..4a68c86461a 100644 --- a/pkg/api/labels.go +++ b/pkg/api/labels.go @@ -31,6 +31,8 @@ const ( ServiceLabel = "com.docker.compose.service" // ConfigHashLabel stores configuration hash for a compose service ConfigHashLabel = "com.docker.compose.config-hash" + // VolumeConfigHashLabel stores configuration hash for a compose volume + VolumeConfigHashLabel = "com.docker.compose.volume-config-hash" // ContainerNumberLabel stores the container index of a replicated service ContainerNumberLabel = "com.docker.compose.container-number" // VolumeLabel allow to track resource related to a compose volume diff --git a/pkg/compose/create.go b/pkg/compose/create.go index f82b3c105bf..92ffa271c64 100644 --- a/pkg/compose/create.go +++ b/pkg/compose/create.go @@ -65,6 +65,8 @@ type createConfigs struct { Links []string } +type ComposeVolumes map[string]*volumetypes.Volume + func (s *composeService) Create(ctx context.Context, project *types.Project, createOpts api.CreateOptions) error { return progress.RunWithTitle(ctx, func(ctx context.Context) error { return s.create(ctx, project, createOpts) @@ -81,12 +83,6 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt return err } - var observedState Containers - observedState, err = s.getContainers(ctx, project.Name, oneOffInclude, true) - if err != nil { - return err - } - err = s.ensureImagesExists(ctx, project, options.Build, options.QuietPull) if err != nil { return err @@ -98,10 +94,15 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt return err } - if err := s.ensureProjectVolumes(ctx, project); err != nil { + if err := s.ensureProjectVolumes(ctx, project, options.RecreateVolumes); err != nil { return err } + var observedState Containers + observedState, err = s.getContainers(ctx, project.Name, oneOffInclude, true) + if err != nil { + return err + } orphans := observedState.filter(isOrphaned(project)) if len(orphans) > 0 && !options.IgnoreOrphans { if options.RemoveOrphans { @@ -119,6 +120,25 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt return newConvergence(options.Services, observedState, s).apply(ctx, project, options) } +func (s *composeService) getVolumes(ctx context.Context, projectName string) (ComposeVolumes, error) { + volumeList, err := s.apiClient().VolumeList(ctx, volumetypes.ListOptions{ + Filters: filters.NewArgs(projectFilter(projectName)), + }) + if err != nil { + return nil, err + } + if volumeList.Volumes == nil { + return nil, nil + } + + volumes := make(ComposeVolumes, len(volumeList.Volumes)) + for _, v := range volumeList.Volumes { + name := v.Labels[api.VolumeLabel] + volumes[name] = v + } + return volumes, nil +} + func prepareNetworks(project *types.Project) { for k, nw := range project.Networks { nw.Labels = nw.Labels.Add(api.NetworkLabel, k) @@ -139,71 +159,151 @@ func (s *composeService) ensureNetworks(ctx context.Context, networks types.Netw return nil } -func (s *composeService) ensureProjectVolumes(ctx context.Context, project *types.Project) error { +func (s *composeService) ensureProjectVolumes(ctx context.Context, project *types.Project, recreateVolumes bool) error { + // create observed state for volumes + observedVolumesState, err := s.getVolumes(ctx, project.Name) + if err != nil { + return err + } for k, volume := range project.Volumes { volume.Labels = volume.Labels.Add(api.VolumeLabel, k) volume.Labels = volume.Labels.Add(api.ProjectLabel, project.Name) volume.Labels = volume.Labels.Add(api.VersionLabel, api.ComposeVersion) - err := s.ensureVolume(ctx, volume, project.Name) + volumeConfigHash, err := VolumeHash(volume) + if err != nil { + return err + } + volume.Labels = volume.Labels.Add(api.VolumeConfigHashLabel, volumeConfigHash) + + err = s.ensureVolume(ctx, volume, project.Name) + if err != nil { + return err + } + + actualVolume, exists := observedVolumesState[k] + if !exists { + continue + } + // Check if it's necessary to recreate the volume by comparing + // with the current volume config hash + shouldRecreateVolume, err := shouldUpdateVolume(volume, actualVolume) if err != nil { return err } + if !shouldRecreateVolume { + continue + } + if !recreateVolumes && shouldRecreateVolume { + logrus.Warnf("Warning: An update has been made to the volume %s. To apply this update to your running Compose project,"+ + "please use the up command with the --recreate-volume option. Note that recreating the volume will remove the existing "+ + "volume in order to update it.", k) + continue + } + if recreateVolumes && shouldRecreateVolume { + err = s.recreateVolume(ctx, project, k, volume) + if err != nil { + return err + } + } } - err := func() error { - if s.manageDesktopFileSharesEnabled(ctx) { - // collect all the bind mount paths and try to set up file shares in - // Docker Desktop for them - var paths []string - for _, svcName := range project.ServiceNames() { - svc := project.Services[svcName] - for _, vol := range svc.Volumes { - if vol.Type != string(mount.TypeBind) { - continue - } - p := filepath.Clean(vol.Source) - if !filepath.IsAbs(p) { - return fmt.Errorf("file share path is not absolute: %s", p) - } - if fi, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) { - // actual directory will be implicitly created when the - // file share is initialized if it doesn't exist, so - // need to filter out any that should not be auto-created - if vol.Bind != nil && !vol.Bind.CreateHostPath { - logrus.Debugf("Skipping creating file share for %q: does not exist and `create_host_path` is false", p) - continue - } - } else if err != nil { - // if we can't read the path, we won't be able to make - // a file share for it - logrus.Debugf("Skipping creating file share for %q: %v", p, err) - continue - } else if !fi.IsDir() { - // ignore files & special types (e.g. Unix sockets) - logrus.Debugf("Skipping creating file share for %q: not a directory", p) + err = s.configureFileShares(ctx, project) + if err != nil { + progress.ContextWriter(ctx).TailMsgf("Failed to prepare Synchronized file shares: %v", err) + } + return nil +} + +func (s *composeService) configureFileShares(ctx context.Context, project *types.Project) error { + if s.manageDesktopFileSharesEnabled(ctx) { + // collect all the bind mount paths and try to set up file shares in + // Docker Desktop for them + var paths []string + for _, svcName := range project.ServiceNames() { + svc := project.Services[svcName] + for _, vol := range svc.Volumes { + if vol.Type != string(mount.TypeBind) { + continue + } + p := filepath.Clean(vol.Source) + if !filepath.IsAbs(p) { + return fmt.Errorf("file share path is not absolute: %s", p) + } + if fi, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) { + // actual directory will be implicitly created when the + // file share is initialized if it doesn't exist, so + // need to filter out any that should not be auto-created + if vol.Bind != nil && !vol.Bind.CreateHostPath { + logrus.Debugf("Skipping creating file share for %q: does not exist and `create_host_path` is false", p) continue } - - paths = append(paths, p) + } else if err != nil { + // if we can't read the path, we won't be able to make + // a file share for it + logrus.Debugf("Skipping creating file share for %q: %v", p, err) + continue + } else if !fi.IsDir() { + // ignore files & special types (e.g. Unix sockets) + logrus.Debugf("Skipping creating file share for %q: not a directory", p) + continue } + + paths = append(paths, p) } + } + + // remove duplicate/unnecessary child paths and sort them for predictability + paths = pathutil.EncompassingPaths(paths) + sort.Strings(paths) - // remove duplicate/unnecessary child paths and sort them for predictability - paths = pathutil.EncompassingPaths(paths) - sort.Strings(paths) + fileShareManager := desktop.NewFileShareManager(s.desktopCli, project.Name, paths) + if err := fileShareManager.EnsureExists(ctx); err != nil { + return fmt.Errorf("initializing file shares: %w", err) + } + } + return nil +} - fileShareManager := desktop.NewFileShareManager(s.desktopCli, project.Name, paths) - if err := fileShareManager.EnsureExists(ctx); err != nil { - return fmt.Errorf("initializing file shares: %w", err) +func (s *composeService) recreateVolume(ctx context.Context, project *types.Project, volumeID string, volumeConfig types.VolumeConfig) error { + // need to recreate containers associated with the volume first + selectedService := []string{} + for _, s := range project.Services { + for _, v := range s.Volumes { + if v.Source == volumeID { + selectedService = append(selectedService, s.Name) } } - return nil - }() + } + err := s.Remove(ctx, project.Name, api.RemoveOptions{ + Project: project, + Services: selectedService, + Force: true, + Stop: true, + }) + if err != nil { + return err + } + // remove volume to update it + w := progress.ContextWriter(ctx) + err = s.removeVolume(ctx, volumeConfig.Name, w) + if err != nil { + return err + } + + // create the volume + return s.createVolume(ctx, volumeConfig) +} +func shouldUpdateVolume(config types.VolumeConfig, volume *volumetypes.Volume) (bool, error) { + previousHash, ok := volume.Labels[api.VolumeConfigHashLabel] + if !ok { + return false, nil + } + currentHash, err := VolumeHash(config) if err != nil { - progress.ContextWriter(ctx).TailMsgf("Failed to prepare Synchronized file shares: %v", err) + return false, fmt.Errorf("could not get hash label for volume config %s", config.Name) } - return nil + return previousHash != currentHash, nil } func (s *composeService) getCreateConfigs(ctx context.Context, diff --git a/pkg/compose/hash.go b/pkg/compose/hash.go index e6a189bd8db..a6881c53ae8 100644 --- a/pkg/compose/hash.go +++ b/pkg/compose/hash.go @@ -20,6 +20,7 @@ import ( "encoding/json" "github.com/compose-spec/compose-go/v2/types" + "github.com/docker/compose/v2/pkg/api" "github.com/opencontainers/go-digest" ) @@ -40,3 +41,34 @@ func ServiceHash(o types.ServiceConfig) (string, error) { } return digest.SHA256.FromBytes(bytes).Encoded(), nil } + +func DeepCopy(src *types.VolumeConfig) (types.VolumeConfig, error) { + var dst types.VolumeConfig + data, err := json.Marshal(src) + if err != nil { + return types.VolumeConfig{}, err + } + err = json.Unmarshal(data, &dst) + return dst, err +} + +// From a top-level Volume Configuration, creates a unique hash ignoring +// External +func VolumeHash(v types.VolumeConfig) (string, error) { + o, err := DeepCopy(&v) + if err != nil { + return "", err + } + + if o.Driver == "" { // (TODO: jhrotko) This probably should be fixed in compose-go + o.Driver = "local" + } + o.External = false // (TODO: jhrotko) the name can change. Need to think about this case + delete(o.Labels, api.VolumeConfigHashLabel) + + bytes, err := json.Marshal(o) + if err != nil { + return "", err + } + return digest.SHA256.FromBytes(bytes).Encoded(), nil +} diff --git a/pkg/compose/kill.go b/pkg/compose/kill.go index 5dfa811f0f8..ce92ec1c289 100644 --- a/pkg/compose/kill.go +++ b/pkg/compose/kill.go @@ -18,7 +18,6 @@ package compose import ( "context" - "fmt" "strings" moby "github.com/docker/docker/api/types" @@ -45,6 +44,11 @@ func (s *composeService) kill(ctx context.Context, projectName string, options a return err } + if len(containers) == 0 { + w.Event(progress.SkippedEvent(strings.Join(services, ","), "No containers for selected service")) + return nil + } + project := options.Project if project == nil { project, err = s.getProjectWithResources(ctx, containers, projectName) @@ -56,10 +60,6 @@ func (s *composeService) kill(ctx context.Context, projectName string, options a if !options.RemoveOrphans { containers = containers.filter(isService(project.ServiceNames()...)) } - if len(containers) == 0 { - _, _ = fmt.Fprintf(s.stdinfo(), "no container to kill") - return nil - } eg, ctx := errgroup.WithContext(ctx) containers. diff --git a/pkg/compose/remove.go b/pkg/compose/remove.go index 3866e5f9a9c..cfd9614d9de 100644 --- a/pkg/compose/remove.go +++ b/pkg/compose/remove.go @@ -82,9 +82,7 @@ func (s *composeService) Remove(ctx context.Context, projectName string, options return nil } msg := fmt.Sprintf("Going to remove %s", strings.Join(names, ", ")) - if options.Force { - _, _ = fmt.Fprintln(s.stdout(), msg) - } else { + if !options.Force { confirm, err := prompt.NewPrompt(s.stdin(), s.stdout()).Confirm(msg, false) if err != nil { return err diff --git a/pkg/e2e/fixtures/recreate-volumes/compose.yaml b/pkg/e2e/fixtures/recreate-volumes/compose.yaml new file mode 100644 index 00000000000..5f4452489e9 --- /dev/null +++ b/pkg/e2e/fixtures/recreate-volumes/compose.yaml @@ -0,0 +1,8 @@ +services: + app: + image: alpine + volumes: + - my_vol:/my_vol + +volumes: + my_vol: \ No newline at end of file diff --git a/pkg/e2e/fixtures/recreate-volumes/compose2.yaml b/pkg/e2e/fixtures/recreate-volumes/compose2.yaml new file mode 100644 index 00000000000..c34e35455ed --- /dev/null +++ b/pkg/e2e/fixtures/recreate-volumes/compose2.yaml @@ -0,0 +1,9 @@ +services: + app: + image: alpine + volumes: + - my_vol:/my_vol + +volumes: + my_vol: + name: cool \ No newline at end of file diff --git a/pkg/e2e/up_test.go b/pkg/e2e/up_test.go index e6e138c3bce..5a7d361255f 100644 --- a/pkg/e2e/up_test.go +++ b/pkg/e2e/up_test.go @@ -179,3 +179,29 @@ func TestUpWithAllResources(t *testing.T) { assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Volume "%s_my_vol" Created`, projectName)), res.Combined()) assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Network %s_my_net Created`, projectName)), res.Combined()) } + +func TestUpRecreateVolumes(t *testing.T) { + c := NewCLI(t) + const projectName = "compose-e2e-recreate-volumes" + t.Cleanup(func() { + c.RunDockerComposeCmd(t, "--project-name", projectName, "down", "-v") + }) + + res := c.RunDockerComposeCmd(t, "-f", "./fixtures/recreate-volumes/compose.yaml", "--project-name", projectName, "up", "-d") + assert.NilError(t, res.Error) + assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Volume "%s_my_vol" Created`, projectName)), res.Combined()) + + res = c.RunDockerComposeCmd(t, "-f", "./fixtures/recreate-volumes/compose.yaml", "--project-name", projectName, "up", "--recreate-volumes", "-d") + // If there are no changes it does not recreate volume + assert.Assert(t, !strings.Contains(res.Combined(), fmt.Sprintf(`Volume "%s_my_vol"`, projectName)), res.Combined()) + + res = c.RunDockerComposeCmd(t, "-f", "./fixtures/recreate-volumes/compose2.yaml", "--project-name", projectName, "up", "--recreate-volumes", "-d") + // removes the affected container + assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Container %s-app-1 Stopped`, projectName)), res.Combined()) + assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Container %s-app-1 Removed`, projectName)), res.Combined()) + // recreates the changed volume + assert.Assert(t, strings.Contains(res.Combined(), "Volume cool Removed"), res.Combined()) + assert.Assert(t, strings.Contains(res.Combined(), `Volume "cool" Created`), res.Combined()) + // starts the container + assert.Assert(t, strings.Contains(res.Combined(), fmt.Sprintf(`Container %s-app-1 Started`, projectName)), res.Combined()) +}