HDDS-15879. [Ozone Versioning] [T1] Metadata foundation

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -180,6 +180,15 @@ public final class OzoneConsts {
 
   public static final String OM_KEY_PREFIX = "/";
   public static final String DOUBLE_SLASH_OM_KEY_PREFIX  = "//";
+  /**
+   * Separates a key name from the versionId suffix in versionedKeyTable DB keys.
+   * OM_KEY_PREFIX cannot be used: OBJECT_STORE key names contain '/' verbatim, so
+   * a key's versions would interleave with the versions of keys nested under it.
+   * 0x00 is the minimum byte value, so {keyName} + this separator can never be a
+   * prefix of {keyName} + "/". Written as an octal escape because a literal
+   * unicode escape for NUL is expanded by the Java lexer before parsing.
+   */
+  public static final String OM_VERSIONED_KEY_SEPARATOR = "\0";
   public static final String OM_USER_PREFIX = "$";
   public static final String OM_S3_PREFIX = "S3:";
   public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|";
@@ -299,6 +308,7 @@ public final class OzoneConsts {
   public static final String STORAGE_TYPE = "storageType";
   public static final String RESOURCE_TYPE = "resourceType";
   public static final String IS_VERSION_ENABLED = "isVersionEnabled";
+  public static final String VERSIONING_STATUS = "versioningStatus";
   public static final String CREATION_TIME = "creationTime";
   public static final String MODIFICATION_TIME = "modificationTime";
   public static final String DATA_SIZE = "dataSize";

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketVersioningStatus.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketVersioningStatusProto;
+
+/**
+ * S3-compatible bucket versioning state machine:
+ * UNVERSIONED -> ENABLED <-> SUSPENDED.
+ * Once versioning has been enabled, a bucket can never return to UNVERSIONED.
+ */
+public enum BucketVersioningStatus {
+  UNVERSIONED,
+  ENABLED,
+  SUSPENDED;
+
+  public static BucketVersioningStatus fromProto(BucketVersioningStatusProto proto) {
+    switch (proto) {
+    case VERSIONING_ENABLED:
+      return ENABLED;
+    case VERSIONING_SUSPENDED:
+      return SUSPENDED;
+    case UNVERSIONED:
+    default:
+      return UNVERSIONED;
+    }
+  }
+
+  public BucketVersioningStatusProto toProto() {
+    switch (this) {
+    case ENABLED:
+      return BucketVersioningStatusProto.VERSIONING_ENABLED;
+    case SUSPENDED:
+      return BucketVersioningStatusProto.VERSIONING_SUSPENDED;
+    default:
+      return BucketVersioningStatusProto.UNVERSIONED;
+    }
+  }
+
+  /** The legacy isVersionEnabled flag value kept in sync with this status. */
+  public boolean toVersionEnabledFlag() {
+    return this == ENABLED;
+  }
+
+  /**
+   * Whether a bucket may transition from this status to {@code target}.
+   * The only forbidden transition is back to UNVERSIONED after versioning
+   * has been enabled or suspended (matches the S3 state machine).
+   */
+  public boolean canTransitionTo(BucketVersioningStatus target) {
+    return this == UNVERSIONED || target != UNVERSIONED;
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java

@@ -44,6 +44,11 @@ public final class OmBucketArgs extends WithMetadata implements Auditable {
    * Bucket Version flag.
    */
   private final Boolean isVersionEnabled;
+  /**
+   * S3-compatible versioning status; null when not being changed.
+   * Takes precedence over isVersionEnabled when both are set.
+   */
+  private final BucketVersioningStatus versioningStatus;
   /**
    * Type of storage to be used for this bucket.
    * [RAM_DISK, SSD, DISK, ARCHIVE]
@@ -73,6 +78,7 @@ private OmBucketArgs(Builder b) {
     this.volumeName = b.volumeName;
     this.bucketName = b.bucketName;
     this.isVersionEnabled = b.isVersionEnabled;
+    this.versioningStatus = b.versioningStatus;
     this.storageType = b.storageType;
     this.ownerName = b.ownerName;
     this.defaultReplicationConfig = b.defaultReplicationConfig;
@@ -108,6 +114,14 @@ public Boolean getIsVersionEnabled() {
     return isVersionEnabled;
   }
 
+  /**
+   * Returns the requested versioning status, or null if not being changed.
+   * @return BucketVersioningStatus
+   */
+  public BucketVersioningStatus getVersioningStatus() {
+    return versioningStatus;
+  }
+
   /**
    * Returns the type of storage to be used.
    * @return StorageType
@@ -190,6 +204,9 @@ public Map<String, String> toAuditMap() {
         getMetadata().get(OzoneConsts.GDPR_FLAG));
     auditMap.put(OzoneConsts.IS_VERSION_ENABLED,
                 String.valueOf(this.isVersionEnabled));
+    if (this.versioningStatus != null) {
+      auditMap.put(OzoneConsts.VERSIONING_STATUS, this.versioningStatus.name());
+    }
     if (this.storageType != null) {
       auditMap.put(OzoneConsts.STORAGE_TYPE, this.storageType.name());
     }
@@ -227,6 +244,7 @@ public static class Builder extends WithMetadata.Builder {
     private String volumeName;
     private String bucketName;
     private Boolean isVersionEnabled;
+    private BucketVersioningStatus versioningStatus;
     private StorageType storageType;
     private boolean quotaInBytesSet = false;
     private long quotaInBytes;
@@ -261,6 +279,11 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
       return this;
     }
 
+    public Builder setVersioningStatus(BucketVersioningStatus status) {
+      this.versioningStatus = status;
+      return this;
+    }
+
     @Deprecated
     public Builder setBucketEncryptionKey(BucketEncryptionKeyInfo info) {
       if (info == null || info.getKeyName() != null) {
@@ -339,6 +362,9 @@ public BucketArgs getProtobuf() {
     if (isVersionEnabled != null) {
       builder.setIsVersionEnabled(isVersionEnabled);
     }
+    if (versioningStatus != null) {
+      builder.setVersioningStatus(versioningStatus.toProto());
+    }
     if (storageType != null) {
       builder.setStorageType(storageType.toProto());
     }
@@ -381,6 +407,10 @@ public static Builder builderFromProtobuf(BucketArgs bucketArgs) {
     if (bucketArgs.hasIsVersionEnabled()) {
       builder.setIsVersionEnabled(bucketArgs.getIsVersionEnabled());
     }
+    if (bucketArgs.hasVersioningStatus()) {
+      builder.setVersioningStatus(
+          BucketVersioningStatus.fromProto(bucketArgs.getVersioningStatus()));
+    }
     if (bucketArgs.hasStorageType()) {
       builder.setStorageType(StorageType.valueOf(bucketArgs.getStorageType()));
     }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java

@@ -59,9 +59,13 @@ public final class OmBucketInfo extends WithObjectID implements Auditable, CopyO
    */
   private final ImmutableList<OzoneAcl> acls;
   /**
-   * Bucket Version flag.
+   * Bucket Version flag, kept in sync with versioningStatus (ENABLED -> true).
    */
   private final boolean isVersionEnabled;
+  /**
+   * S3-compatible versioning status; authoritative over isVersionEnabled.
+   */
+  private final BucketVersioningStatus versioningStatus;
   /**
    * Type of storage to be used for this bucket.
    * [RAM_DISK, SSD, DISK, ARCHIVE]
@@ -118,7 +122,14 @@ private OmBucketInfo(Builder b) {
     this.volumeName = b.volumeName;
     this.bucketName = b.bucketName;
     this.acls = b.acls.build();
-    this.isVersionEnabled = b.isVersionEnabled;
+    // A null versioningStatus means the bucket carries no S3 versioning status
+    // at all, mirroring the optional proto field: such a bucket is driven by the
+    // legacy isVersionEnabled flag alone. The flag is derived from the status
+    // only when a status was actually set, so that a legacy client enabling
+    // versioning does not silently opt an existing bucket into S3 versioning.
+    this.versioningStatus = b.versioningStatus;
+    this.isVersionEnabled = b.versioningStatus != null
+        ? b.versioningStatus.toVersionEnabledFlag() : b.isVersionEnabled;
     this.storageType = b.storageType;
     this.creationTime = b.creationTime;
     this.modificationTime = b.modificationTime;
@@ -173,6 +184,27 @@ public boolean getIsVersionEnabled() {
     return isVersionEnabled;
   }
 
+  /**
+   * Returns the S3-compatible versioning status; never null. A bucket that only
+   * carries the legacy isVersionEnabled flag is UNVERSIONED as far as S3
+   * versioning is concerned: the legacy flag selects the in-record block version
+   * list, which is a different feature.
+   * @return BucketVersioningStatus
+   */
+  public BucketVersioningStatus getVersioningStatus() {
+    return versioningStatus != null
+        ? versioningStatus : BucketVersioningStatus.UNVERSIONED;
+  }
+
+  /**
+   * Whether an S3 versioning status was explicitly set on this bucket, as
+   * opposed to the bucket carrying only the legacy isVersionEnabled flag.
+   * @return whether the optional status field is present
+   */
+  public boolean hasVersioningStatus() {
+    return versioningStatus != null;
+  }
+
   /**
    * Returns the type of storage to be used.
    * @return StorageType
@@ -338,6 +370,8 @@ public Map<String, String> toAuditMap() {
         (this.acls != null) ? this.acls.toString() : null);
     auditMap.put(OzoneConsts.IS_VERSION_ENABLED,
         String.valueOf(this.isVersionEnabled));
+    auditMap.put(OzoneConsts.VERSIONING_STATUS,
+        this.versioningStatus != null ? this.versioningStatus.name() : null);
     auditMap.put(OzoneConsts.STORAGE_TYPE,
         (this.storageType != null) ? this.storageType.name() : null);
     auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
@@ -379,6 +413,7 @@ public Builder toBuilder() {
         .setBucketName(bucketName)
         .setStorageType(storageType)
         .setIsVersionEnabled(isVersionEnabled)
+        .setVersioningStatus(versioningStatus)
         .setCreationTime(creationTime)
         .setModificationTime(modificationTime)
         .setBucketEncryptionKey(bekInfo)
@@ -408,6 +443,11 @@ public OmBucketInfo withOperationalPropertiesFrom(OmBucketInfo source) {
     return toBuilder()
         .setStorageType(source.getStorageType())
         .setIsVersionEnabled(source.getIsVersionEnabled())
+        // Only when the real bucket actually carries a status: copying the
+        // value getVersioningStatus() derives for a legacy bucket would give
+        // the link an explicit status the real bucket does not have.
+        .setVersioningStatus(source.hasVersioningStatus()
+            ? source.getVersioningStatus() : null)
         .setBucketEncryptionKey(source.getEncryptionKeyInfo())
         .setUsedBytes(source.getUsedBytes())
         .setUsedNamespace(source.getUsedNamespace())
@@ -430,6 +470,7 @@ public static class Builder extends WithObjectID.Builder<OmBucketInfo> {
     private String bucketName;
     private final AclListBuilder acls;
     private boolean isVersionEnabled;
+    private BucketVersioningStatus versioningStatus;
     private StorageType storageType = StorageType.DISK;
     private long creationTime;
     private long modificationTime;
@@ -486,11 +527,26 @@ public Builder addAcl(OzoneAcl ozoneAcl) {
       return this;
     }
 
+    /**
+     * Sets the legacy flag only. It deliberately does not derive a
+     * versioningStatus: a legacy client enabling versioning must not opt the
+     * bucket into S3 versioning semantics. Deriving the status is the job of
+     * OMBucketSetPropertyRequest, where an actual state transition is requested.
+     */
     public Builder setIsVersionEnabled(boolean versionFlag) {
       this.isVersionEnabled = versionFlag;
       return this;
     }
 
+    /** No-op when status is null (e.g. records without the new field). */
+    public Builder setVersioningStatus(BucketVersioningStatus status) {
+      if (status != null) {
+        this.versioningStatus = status;
+        this.isVersionEnabled = status.toVersionEnabledFlag();
+      }
+      return this;
+    }
+
     public Builder setStorageType(StorageType storage) {
       this.storageType = storage;
       return this;
@@ -639,6 +695,12 @@ public BucketInfo getProtobuf() {
         .setQuotaInNamespace(quotaInNamespace)
         .setSnapshotUsedBytes(snapshotUsedBytes)
         .setSnapshotUsedNamespace(snapshotUsedNamespace);
+    // Written only when actually set, so that hasVersioningStatus() keeps
+    // telling a legacy-flag bucket apart from an S3-versioned one after a
+    // round trip through RocksDB.
+    if (versioningStatus != null) {
+      bib.setVersioningStatus(versioningStatus.toProto());
+    }
     if (bucketLayout != null) {
       bib.setBucketLayout(bucketLayout.toProto());
     }
@@ -683,6 +745,9 @@ public static Builder builderFromProtobuf(BucketInfo bucketInfo,
         .setAcls(bucketInfo.getAclsList().stream().map(
             OzoneAcl::fromProtobuf).collect(Collectors.toList()))
         .setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
+        .setVersioningStatus(bucketInfo.hasVersioningStatus()
+            ? BucketVersioningStatus.fromProto(bucketInfo.getVersioningStatus())
+            : null)
         .setStorageType(StorageType.valueOf(bucketInfo.getStorageType()))
         .setCreationTime(bucketInfo.getCreationTime())
         .setUsedBytes(bucketInfo.getUsedBytes())
@@ -789,6 +854,7 @@ public boolean equals(Object o) {
         bucketName.equals(that.bucketName) &&
         Objects.equals(acls, that.acls) &&
         Objects.equals(isVersionEnabled, that.isVersionEnabled) &&
+        versioningStatus == that.versioningStatus &&
         storageType == that.storageType &&
         getObjectID() == that.getObjectID() &&
         getUpdateID() == that.getUpdateID() &&
@@ -817,6 +883,7 @@ public String toString() {
         ", bucketName='" + bucketName + "'" +
         ", acls=" + acls +
         ", isVersionEnabled=" + isVersionEnabled +
+        ", versioningStatus=" + versioningStatus +
         ", storageType=" + storageType +
         ", creationTime=" + creationTime +
         ", bekInfo=" + bekInfo +