HDDS-16060. [Ozone Versioning] [T7] Snapshot exclusion

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -180,6 +180,15 @@ public final class OzoneConsts {
 
   public static final String OM_KEY_PREFIX = "/";
   public static final String DOUBLE_SLASH_OM_KEY_PREFIX  = "//";
+  /**
+   * Separates a key name from the versionId suffix in versionedKeyTable DB keys.
+   * OM_KEY_PREFIX cannot be used: OBJECT_STORE key names contain '/' verbatim, so
+   * a key's versions would interleave with the versions of keys nested under it.
+   * 0x00 is the minimum byte value, so {keyName} + this separator can never be a
+   * prefix of {keyName} + "/". Written as an octal escape because a literal
+   * unicode escape for NUL is expanded by the Java lexer before parsing.
+   */
+  public static final String OM_VERSIONED_KEY_SEPARATOR = "\0";
   public static final String OM_USER_PREFIX = "$";
   public static final String OM_S3_PREFIX = "S3:";
   public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|";
@@ -299,6 +308,14 @@ public final class OzoneConsts {
   public static final String STORAGE_TYPE = "storageType";
   public static final String RESOURCE_TYPE = "resourceType";
   public static final String IS_VERSION_ENABLED = "isVersionEnabled";
+  public static final String VERSIONING_STATUS = "versioningStatus";
+  // Audit keys for the object versions a write creates, demotes or removes.
+  public static final String VERSION_ID = "versionId";
+  public static final String REQUESTED_VERSION_ID = "requestedVersionId";
+  public static final String SUPERSEDED_VERSION_ID = "supersededVersionId";
+  public static final String DELETED_VERSION_ID = "deletedVersionId";
+  public static final String PROMOTED_VERSION_ID = "promotedVersionId";
+  public static final String NULL_VERSION = "nullVersion";
   public static final String CREATION_TIME = "creationTime";
   public static final String MODIFICATION_TIME = "modificationTime";
   public static final String DATA_SIZE = "dataSize";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -5216,9 +5216,11 @@
 
   <property>
     <name>ozone.lifecycle.service.enabled</name>
-    <value>false</value>
+    <value>true</value>
     <tag>OZONE</tag>
-    <description>It specifies whether to enable lifecycle management service.</description>
+    <description>It specifies whether to enable lifecycle management service.
+      S3 object versioning depends on it to reclaim versions, so OM refuses to
+      enable versioning on a bucket while this is off.</description>
   </property>
   <property>
     <name>ozone.lifecycle.service.move.to.trash.enabled</name>
@@ -5286,4 +5288,52 @@
       ozone.lifecycle.service.state.save.keys.processed is satisfied.
     </description>
   </property>
+  <property>
+    <name>ozone.om.versioning.version-id-generator</name>
+    <value>org.apache.hadoop.ozone.om.helpers.UniqueIdVersionIdGenerator</value>
+    <tag>OZONE, OM</tag>
+    <description>
+      Implementation of org.apache.hadoop.ozone.om.helpers.VersionIdGenerator used to propose the
+      versionId of an object version. The default numbers a version with the time it was written,
+      through the same scheme Ozone uses for block local IDs;
+      org.apache.hadoop.ozone.om.helpers.PinnedFirstVersionIdGenerator additionally pins the first
+      version of every key to a reserved sentinel, so that it can be referenced without listing the
+      key's versions first.
+      The setting is cluster-wide and read when an OM starts, so a change takes effect on every
+      bucket once the OMs have been restarted; it is not recorded in bucket metadata. Keys written
+      under a previous generator keep the ids they were given: a proposal is taken as a floor when
+      the version is applied, so a generator proposing ids below those a previous one handed out
+      delays the new numbering on the affected keys rather than disordering them.
+    </description>
+  </property>
+  <property>
+    <name>ozone.om.versioning.lookup.warning.threshold</name>
+    <value>5s</value>
+    <tag>OZONE, OM, PERFORMANCE</tag>
+    <description>
+      How long a lookup of a key's noncurrent versions may take on the write path before it is
+      logged at WARN. The lookup walks the versionedKeyTable cache and opens an iterator on the DB,
+      both inline on the write, and creating a RocksDB iterator can be slow under load. The latency
+      is also reported as an OM performance metric regardless of this threshold.
+    </description>
+  </property>
+  <property>
+    <name>ozone.om.snapshot.versioning.coexistence.enabled</name>
+    <value>false</value>
+    <tag>OZONE, OM</tag>
+    <description>
+      UNSAFE, FOR TESTING ONLY. Do not enable on a production cluster.
+
+      When false, the default, OM refuses to create a snapshot of a bucket that has S3 object
+      versioning enabled or suspended, and refuses to enable or resume versioning on a bucket that
+      still has snapshots. The two features cannot be combined safely yet: a snapshot shares blocks
+      with the active object store and keeps them only because key reclamation skips what a previous
+      snapshot references, and that lookup is not version-aware, so reclaiming a noncurrent version
+      can delete blocks a snapshot still points at.
+
+      Setting this to true lifts those checks so that the integration tests for snapshot-aware
+      version reclamation can be written against a real mixed-state bucket. On such a bucket, data
+      loss is possible.
+    </description>
+  </property>
 </configuration>

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneLifecycleConfiguration.java

@@ -47,10 +47,21 @@ public OzoneLifecycleConfiguration(String volume, String bucket,
   public static class OzoneLCExpiration {
     private final Integer days;
     private final String date;
+    private final boolean expiredObjectDeleteMarker;
 
     public OzoneLCExpiration(Integer days, String date) {
+      this(days, date, false);
+    }
+
+    public OzoneLCExpiration(Integer days, String date,
+        boolean expiredObjectDeleteMarker) {
       this.days = days;
       this.date = date;
+      this.expiredObjectDeleteMarker = expiredObjectDeleteMarker;
+    }
+
+    public boolean isExpiredObjectDeleteMarker() {
+      return expiredObjectDeleteMarker;
     }
 
     public String getDate() {
@@ -62,6 +73,28 @@ public Integer getDays() {
     }
   }
 
+  /**
+   * A class that encapsulates OzoneLCNoncurrentVersionExpiration.
+   */
+  public static class OzoneLCNoncurrentVersionExpiration {
+    private final Integer noncurrentDays;
+    private final Integer newerNoncurrentVersions;
+
+    public OzoneLCNoncurrentVersionExpiration(Integer noncurrentDays,
+        Integer newerNoncurrentVersions) {
+      this.noncurrentDays = noncurrentDays;
+      this.newerNoncurrentVersions = newerNoncurrentVersions;
+    }
+
+    public Integer getNoncurrentDays() {
+      return noncurrentDays;
+    }
+
+    public Integer getNewerNoncurrentVersions() {
+      return newerNoncurrentVersions;
+    }
+  }
+
   /**
    * A class that encapsulates OzoneLCAbortIncompleteMultipartUpload.
    */
@@ -136,19 +169,34 @@ public static class OzoneLCRule {
     private final String status;
     private final OzoneLCExpiration expiration;
     private final OzoneLCAbortIncompleteMultipartUpload abortIncompleteMultipartUpload;
+    private final OzoneLCNoncurrentVersionExpiration noncurrentVersionExpiration;
     private final OzoneLCFilter filter;
 
     public OzoneLCRule(String id, String prefix, String status,
         OzoneLCExpiration expiration, OzoneLCAbortIncompleteMultipartUpload abortIncompleteMultipartUpload,
         OzoneLCFilter filter) {
+      this(id, prefix, status, expiration, abortIncompleteMultipartUpload, null,
+          filter);
+    }
+
+    @SuppressWarnings("checkstyle:ParameterNumber")
+    public OzoneLCRule(String id, String prefix, String status,
+        OzoneLCExpiration expiration, OzoneLCAbortIncompleteMultipartUpload abortIncompleteMultipartUpload,
+        OzoneLCNoncurrentVersionExpiration noncurrentVersionExpiration,
+        OzoneLCFilter filter) {
       this.id = id;
       this.prefix = prefix;
       this.status = status;
       this.expiration = expiration;
       this.abortIncompleteMultipartUpload = abortIncompleteMultipartUpload;
+      this.noncurrentVersionExpiration = noncurrentVersionExpiration;
       this.filter = filter;
     }
 
+    public OzoneLCNoncurrentVersionExpiration getNoncurrentVersionExpiration() {
+      return noncurrentVersionExpiration;
+    }
+
     public String getId() {
       return id;
     }
@@ -200,7 +248,15 @@ public static OzoneLifecycleConfiguration fromOmLifecycleConfiguration(
       OzoneLifecycleConfiguration.OzoneLCExpiration e = null;
       if (r.getExpiration() != null) {
         e = new OzoneLifecycleConfiguration.OzoneLCExpiration(
-            r.getExpiration().getDays(), r.getExpiration().getDate());
+            r.getExpiration().getDays(), r.getExpiration().getDate(),
+            r.getExpiration().isExpiredObjectDeleteMarker());
+      }
+
+      OzoneLifecycleConfiguration.OzoneLCNoncurrentVersionExpiration n = null;
+      if (r.getNoncurrentVersionExpiration() != null) {
+        n = new OzoneLifecycleConfiguration.OzoneLCNoncurrentVersionExpiration(
+            r.getNoncurrentVersionExpiration().getNoncurrentDays(),
+            r.getNoncurrentVersionExpiration().getNewerNoncurrentVersions());
       }
 
       OzoneLifecycleConfiguration.OzoneLCAbortIncompleteMultipartUpload a = null;
@@ -221,7 +277,7 @@ public static OzoneLifecycleConfiguration fromOmLifecycleConfiguration(
       }
 
       rules.add(new OzoneLifecycleConfiguration.OzoneLCRule(r.getId(),
-          r.getPrefix(), (r.isEnabled() ? "Enabled" : "Disabled"), e, a, f));
+          r.getPrefix(), (r.isEnabled() ? "Enabled" : "Disabled"), e, a, n, f));
     }
 
     return new OzoneLifecycleConfiguration(lifecycleConfiguration.getVolume(),

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -333,6 +333,7 @@ public static boolean isReadOnly(OMRequest omRequest) {
     case DeleteLifecycleConfiguration:
     case SetLifecycleServiceStatus:
     case SaveLifecycleScanState:
+    case ReclaimObjectVersions:
       return false;
     case UnknownCommand:
       return false;
@@ -474,6 +475,9 @@ public static boolean shouldSendToFollower(OMRequest omRequest) {
     case DeleteLifecycleConfiguration:
     case SetLifecycleServiceStatus:
     case SaveLifecycleScanState:
+    // Submitted by the leader's own lifecycle scan, like the scan state it
+    // travels with.
+    case ReclaimObjectVersions:
     case UnknownCommand:
       return false;
     case EchoRPC:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -21,6 +21,8 @@
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.UniqueIdVersionIdGenerator;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
 import org.apache.ratis.util.TimeDuration;
 import org.rocksdb.CompactRangeOptions.BottommostLevelCompaction;
 
@@ -188,7 +190,9 @@ public final class OMConfigKeys {
       = 5;
   public static final String OZONE_KEY_LIFECYCLE_SERVICE_ENABLED =
       "ozone.lifecycle.service.enabled";
-  public static final boolean OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT = false;
+  // On by default: S3 object versioning relies on the lifecycle scan to
+  // reclaim versions, and a rule that never runs is worse than no rule.
+  public static final boolean OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT = true;
   public static final String OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE =
       "ozone.lifecycle.service.delete.batch-size";
   public static final int OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT = 1000;
@@ -768,6 +772,33 @@ public final class OMConfigKeys {
       "ozone.om.ratis.events.max.limit";
   public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
 
+  public static final String OZONE_OM_VERSIONING_VERSION_ID_GENERATOR =
+      "ozone.om.versioning.version-id-generator";
+  public static final Class<? extends VersionIdGenerator>
+      OZONE_OM_VERSIONING_VERSION_ID_GENERATOR_DEFAULT = UniqueIdVersionIdGenerator.class;
+
+  /**
+   * How long a lookup of a key's noncurrent versions may take on the write
+   * path before it is logged. The lookup walks the versionedKeyTable cache
+   * and opens a DB iterator, which can be slow to create under load.
+   */
+  public static final String
+      OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD =
+      "ozone.om.versioning.lookup.warning.threshold";
+  public static final String
+      OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD_DEFAULT = "5s";
+
+  /**
+   * Lifts the mutual exclusion between snapshots and object versioning on a
+   * bucket. Unsafe: a version reclaimed from the active object store can take
+   * blocks a snapshot still references. For tests only.
+   */
+  public static final String
+      OZONE_OM_SNAPSHOT_VERSIONING_COEXISTENCE_ENABLED =
+      "ozone.om.snapshot.versioning.coexistence.enabled";
+  public static final boolean
+      OZONE_OM_SNAPSHOT_VERSIONING_COEXISTENCE_ENABLED_DEFAULT = false;
+
   /**
    * Never constructed.
    */

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java

@@ -283,6 +283,14 @@ public enum ResultCodes {
     ATOMIC_WRITE_CONFLICT,
 
     LIFECYCLE_CONFIGURATION_NOT_FOUND,
-    UPDATE_ID_NOT_MATCH
+    UPDATE_ID_NOT_MATCH,
+
+    /**
+     * The addressed version exists but is a delete marker. Kept distinct from
+     * KEY_NOT_FOUND so that the S3 Gateway can answer such a read with 405
+     * MethodNotAllowed rather than 404; the gateway mapping itself is not in
+     * place yet, and until it is the condition surfaces as an internal error.
+     */
+    KEY_IS_DELETE_MARKER
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketVersioningStatus.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketVersioningStatusProto;
+
+/**
+ * S3-compatible bucket versioning state machine:
+ * UNVERSIONED -> ENABLED <-> SUSPENDED.
+ * Once versioning has been enabled, a bucket can never return to UNVERSIONED.
+ */
+public enum BucketVersioningStatus {
+  UNVERSIONED,
+  ENABLED,
+  SUSPENDED;
+
+  public static BucketVersioningStatus fromProto(BucketVersioningStatusProto proto) {
+    switch (proto) {
+    case VERSIONING_ENABLED:
+      return ENABLED;
+    case VERSIONING_SUSPENDED:
+      return SUSPENDED;
+    case UNVERSIONED:
+    default:
+      return UNVERSIONED;
+    }
+  }
+
+  public BucketVersioningStatusProto toProto() {
+    switch (this) {
+    case ENABLED:
+      return BucketVersioningStatusProto.VERSIONING_ENABLED;
+    case SUSPENDED:
+      return BucketVersioningStatusProto.VERSIONING_SUSPENDED;
+    default:
+      return BucketVersioningStatusProto.UNVERSIONED;
+    }
+  }
+
+  /** The legacy isVersionEnabled flag value kept in sync with this status. */
+  public boolean toVersionEnabledFlag() {
+    return this == ENABLED;
+  }
+
+  /**
+   * Whether a bucket may transition from this status to {@code target}.
+   * The only forbidden transition is back to UNVERSIONED after versioning
+   * has been enabled or suspended (matches the S3 state machine).
+   */
+  public boolean canTransitionTo(BucketVersioningStatus target) {
+    return this == UNVERSIONED || target != UNVERSIONED;
+  }
+}