tags;
private Long expectedDataGeneration = null;
@@ -290,6 +340,8 @@ public Builder(OmKeyArgs obj) {
obj.forceUpdateContainerCacheFromSCM;
this.expectedDataGeneration = obj.expectedDataGeneration;
this.expectedETag = obj.expectedETag;
+ this.versionId = obj.versionId;
+ this.nullVersion = obj.nullVersion;
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
}
@@ -410,6 +462,24 @@ public Builder setRecursive(boolean isRecursive) {
return this;
}
+ /** Addresses the version with this id; clears any null-version selection. */
+ public Builder setVersionId(Long id) {
+ this.versionId = id;
+ if (id != null) {
+ this.nullVersion = false;
+ }
+ return this;
+ }
+
+ /** Addresses the key's null version slot; clears any addressed versionId. */
+ public Builder setNullVersion(boolean isNullVersion) {
+ this.nullVersion = isNullVersion;
+ if (isNullVersion) {
+ this.versionId = null;
+ }
+ return this;
+ }
+
public Builder setHeadOp(boolean isHeadOp) {
this.headOp = isHeadOp;
return this;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index ab4da4badd90..f58edad0c6d8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -112,6 +112,17 @@ public final class OmKeyInfo extends WithParentObjectId
// been modified.
private final Long expectedDataGeneration;
+ // S3-compatible object versioning. versionId is allocated once when a
+ // version is written and then frozen, and increases within a key so its
+ // versions can be ordered newest first; absent on records that predate
+ // versioning support (treated as the null version). A delete marker has
+ // isDeleteMarker set and no data blocks. isNullVersion marks the single
+ // overwritable "null version" slot per key.
+ private final Long versionId;
+ private final boolean isDeleteMarker;
+ private final boolean isNullVersion;
+ private final long noncurrentTime;
+
private OmKeyInfo(Builder b) {
super(b);
this.volumeName = b.volumeName;
@@ -130,6 +141,10 @@ private OmKeyInfo(Builder b) {
this.ownerName = b.ownerName;
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
+ this.versionId = b.versionId;
+ this.isDeleteMarker = b.isDeleteMarker;
+ this.isNullVersion = b.isNullVersion;
+ this.noncurrentTime = b.noncurrentTime;
}
/**
@@ -469,6 +484,50 @@ public FileChecksum getFileChecksum() {
return fileChecksum;
}
+ /**
+ * @return the object version identity, or null for records that predate
+ * versioning support (treated as the null version).
+ */
+ public Long getVersionId() {
+ return versionId;
+ }
+
+ public boolean isDeleteMarker() {
+ return isDeleteMarker;
+ }
+
+ public boolean isNullVersion() {
+ return isNullVersion;
+ }
+
+ /**
+ * When this record stopped being the key's current version, i.e. when the
+ * version that superseded it was written. Stamped as the record is moved
+ * into the versionedKeyTable, so a current version has none.
+ *
+ * This is the clock S3 counts NoncurrentDays from, and it is stored
+ * rather than derived from the neighbouring versions because removing an
+ * intervening version must not move it.
+ *
+ * @return the moment this version became noncurrent, or 0 if it never has
+ */
+ public long getNoncurrentTime() {
+ return noncurrentTime;
+ }
+
+ /**
+ * Whether this record is the key's null version, i.e. what a request naming
+ * version "null" addresses. That is either a record written while versioning
+ * was suspended, which carries the flag, or a record that predates versioning
+ * and so carries no versionId at all: enabling versioning does not rewrite
+ * existing objects, and S3 reports their version as "null".
+ *
+ * @return whether version "null" of the key addresses this record
+ */
+ public boolean isNullVersionRecord() {
+ return isNullVersion || versionId == null;
+ }
+
@Override
public String toString() {
return "OmKeyInfo{" +
@@ -485,6 +544,10 @@ public String toString() {
", isFile=" + isFile +
", fileName='" + fileName + '\'' +
", acls=" + acls +
+ ", versionId=" + versionId +
+ ", isDeleteMarker=" + isDeleteMarker +
+ ", isNullVersion=" + isNullVersion +
+ ", noncurrentTime=" + noncurrentTime +
'}';
}
@@ -511,6 +574,10 @@ public static class Builder extends WithParentObjectId.Builder {
private boolean isFile;
private final MapBuilder tags;
private Long expectedDataGeneration = null;
+ private Long versionId = null;
+ private boolean isDeleteMarker;
+ private boolean isNullVersion;
+ private long noncurrentTime;
public Builder() {
this.acls = AclListBuilder.empty();
@@ -533,6 +600,10 @@ public Builder(OmKeyInfo obj) {
this.fileChecksum = obj.fileChecksum;
this.isFile = obj.isFile;
this.expectedDataGeneration = obj.expectedDataGeneration;
+ this.versionId = obj.versionId;
+ this.isDeleteMarker = obj.isDeleteMarker;
+ this.isNullVersion = obj.isNullVersion;
+ this.noncurrentTime = obj.noncurrentTime;
this.tags = MapBuilder.of(obj.tags);
obj.keyLocationVersions.forEach(keyLocationVersion ->
this.omKeyLocationInfoGroups.add(
@@ -704,6 +775,26 @@ public Builder setExpectedDataGeneration(Long existingGeneration) {
return this;
}
+ public Builder setVersionId(Long versionId) {
+ this.versionId = versionId;
+ return this;
+ }
+
+ public Builder setDeleteMarker(boolean deleteMarker) {
+ this.isDeleteMarker = deleteMarker;
+ return this;
+ }
+
+ public Builder setNoncurrentTime(long noncurrentTime) {
+ this.noncurrentTime = noncurrentTime;
+ return this;
+ }
+
+ public Builder setNullVersion(boolean nullVersion) {
+ this.isNullVersion = nullVersion;
+ return this;
+ }
+
@Override
protected void validate() {
super.validate();
@@ -855,6 +946,19 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName,
if (ownerName != null) {
kb.setOwnerName(ownerName);
}
+ if (versionId != null) {
+ kb.setVersionId(versionId);
+ }
+ // only persisted when set, to keep records without versioning unchanged
+ if (isDeleteMarker) {
+ kb.setIsDeleteMarker(true);
+ }
+ if (isNullVersion) {
+ kb.setIsNullVersion(true);
+ }
+ if (noncurrentTime > 0) {
+ kb.setNoncurrentTime(noncurrentTime);
+ }
return kb.build();
}
@@ -909,6 +1013,18 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) {
if (keyInfo.hasOwnerName()) {
builder.setOwnerName(keyInfo.getOwnerName());
}
+ if (keyInfo.hasVersionId()) {
+ builder.setVersionId(keyInfo.getVersionId());
+ }
+ if (keyInfo.hasIsDeleteMarker()) {
+ builder.setDeleteMarker(keyInfo.getIsDeleteMarker());
+ }
+ if (keyInfo.hasIsNullVersion()) {
+ builder.setNullVersion(keyInfo.getIsNullVersion());
+ }
+ if (keyInfo.hasNoncurrentTime()) {
+ builder.setNoncurrentTime(keyInfo.getNoncurrentTime());
+ }
return builder;
}
@@ -946,7 +1062,11 @@ public boolean isKeyInfoSame(OmKeyInfo omKeyInfo, boolean checkPath,
Objects.equals(getMetadata(), omKeyInfo.getMetadata()) &&
Objects.equals(acls, omKeyInfo.acls) &&
Objects.equals(getTags(), omKeyInfo.getTags()) &&
- getObjectID() == omKeyInfo.getObjectID();
+ getObjectID() == omKeyInfo.getObjectID() &&
+ Objects.equals(versionId, omKeyInfo.versionId) &&
+ isDeleteMarker == omKeyInfo.isDeleteMarker &&
+ isNullVersion == omKeyInfo.isNullVersion &&
+ noncurrentTime == omKeyInfo.noncurrentTime;
if (isEqual && checkUpdateID) {
isEqual = getUpdateID() == omKeyInfo.getUpdateID();
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java
index 3d746bd1381b..44d962c38694 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java
@@ -54,6 +54,7 @@ public interface OmLCAction {
enum ActionType {
EXPIRATION,
ABORT_INCOMPLETE_MULTIPART_UPLOAD,
+ NONCURRENT_VERSION_EXPIRATION,
// Future action types can be added here (e.g., TRANSITION)
}
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java
index 69db359efa60..bb695c330361 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java
@@ -38,6 +38,7 @@
public final class OmLCExpiration implements OmLCAction {
private final Integer days;
private final String date;
+ private final boolean expiredObjectDeleteMarker;
private ZonedDateTime zonedDateTime;
private long daysInMilli;
private static boolean test;
@@ -49,6 +50,7 @@ private OmLCExpiration() {
private OmLCExpiration(Builder builder) {
this.days = builder.days;
this.date = builder.date;
+ this.expiredObjectDeleteMarker = builder.expiredObjectDeleteMarker;
}
@Nullable
@@ -61,6 +63,14 @@ public String getDate() {
return date;
}
+ /**
+ * @return whether the action removes a delete marker that is the key's only
+ * remaining version, rather than expiring an object by age.
+ */
+ public boolean isExpiredObjectDeleteMarker() {
+ return expiredObjectDeleteMarker;
+ }
+
public boolean isExpired(long timestamp) {
if (zonedDateTime != null) {
Instant instant = Instant.ofEpochMilli(timestamp);
@@ -95,9 +105,14 @@ public void valid(long creationTime) throws OMException {
boolean hasDays = days != null;
boolean hasDate = !StringUtils.isBlank(date);
- if (hasDays == hasDate) {
- throw new OMException("Invalid lifecycle configuration: Either 'days' or 'date' " +
- "should be specified, but not both or neither.", OMException.ResultCodes.INVALID_REQUEST);
+ // ExpiredObjectDeleteMarker removes a marker rather than expiring an
+ // object by age, so it cannot be combined with either age.
+ int specified = (hasDays ? 1 : 0) + (hasDate ? 1 : 0)
+ + (expiredObjectDeleteMarker ? 1 : 0);
+ if (specified != 1) {
+ throw new OMException("Invalid lifecycle configuration: exactly one of "
+ + "'Days', 'Date' and 'ExpiredObjectDeleteMarker' should be "
+ + "specified.", OMException.ResultCodes.INVALID_REQUEST);
}
if (hasDays) {
if (days <= 0) {
@@ -161,6 +176,9 @@ public LifecycleAction getProtobuf() {
if (days != null) {
builder.setDays(days);
}
+ if (expiredObjectDeleteMarker) {
+ builder.setExpiredObjectDeleteMarker(true);
+ }
return LifecycleAction.newBuilder().setExpiration(builder).build();
}
@@ -171,6 +189,10 @@ public static OmLCExpiration getFromProtobuf(LifecycleExpiration lifecycleExpira
if (lifecycleExpiration.hasDate()) {
builder.setDate(lifecycleExpiration.getDate());
}
+ if (lifecycleExpiration.hasExpiredObjectDeleteMarker()) {
+ builder.setExpiredObjectDeleteMarker(
+ lifecycleExpiration.getExpiredObjectDeleteMarker());
+ }
if (lifecycleExpiration.hasDays()) {
builder.setDays(lifecycleExpiration.getDays());
}
@@ -183,6 +205,7 @@ public String toString() {
return "OmLCExpiration{" +
"days=" + days +
", date='" + date + '\'' +
+ ", expiredObjectDeleteMarker=" + expiredObjectDeleteMarker +
'}';
}
@@ -192,6 +215,12 @@ public String toString() {
public static class Builder {
private Integer days = null;
private String date = null;
+ private boolean expiredObjectDeleteMarker = false;
+
+ public Builder setExpiredObjectDeleteMarker(boolean removeMarker) {
+ this.expiredObjectDeleteMarker = removeMarker;
+ return this;
+ }
public Builder setDays(int lcDays) {
this.days = lcDays;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCNoncurrentVersionExpiration.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCNoncurrentVersionExpiration.java
new file mode 100644
index 000000000000..f7033f220ff3
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCNoncurrentVersionExpiration.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import jakarta.annotation.Nullable;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.NoncurrentVersionExpiration;
+
+/**
+ * Reclaims the noncurrent versions of a key on a bucket with S3-compatible
+ * versioning, by age, by count, or by both.
+ *
+ * The current version is never touched: it is what a plain read returns, and
+ * expiring it is the {@link OmLCExpiration} action's job. Only versions the
+ * versionedKeyTable holds are candidates.
+ */
+public final class OmLCNoncurrentVersionExpiration implements OmLCAction {
+
+ /** S3's ceiling on NewerNoncurrentVersions. */
+ public static final int MAX_NEWER_NONCURRENT_VERSIONS = 100;
+
+ private final Integer noncurrentDays;
+ private final Integer newerNoncurrentVersions;
+ /**
+ * Derived from noncurrentDays alone, so it is settled here rather than in
+ * {@link #valid()}: an instance whose validation has not run yet would
+ * otherwise read as zero, and every version would be past a rule it is not
+ * past.
+ */
+ private final long noncurrentDaysInMillis;
+
+ private OmLCNoncurrentVersionExpiration(Builder builder) {
+ this.noncurrentDays = builder.noncurrentDays;
+ this.newerNoncurrentVersions = builder.newerNoncurrentVersions;
+ this.noncurrentDaysInMillis = noncurrentDays == null ? 0L
+ : TimeUnit.DAYS.toMillis(noncurrentDays);
+ }
+
+ @Nullable
+ public Integer getNoncurrentDays() {
+ return noncurrentDays;
+ }
+
+ @Nullable
+ public Integer getNewerNoncurrentVersions() {
+ return newerNoncurrentVersions;
+ }
+
+ /**
+ * @param becameNoncurrentAt when the version stopped being current
+ * @return whether it has been noncurrent longer than the rule allows. Always
+ * false when the rule only limits how many versions are kept.
+ */
+ public boolean isExpired(long becameNoncurrentAt) {
+ return noncurrentDays != null
+ && System.currentTimeMillis() > becameNoncurrentAt + noncurrentDaysInMillis;
+ }
+
+ @Override
+ public ActionType getActionType() {
+ return ActionType.NONCURRENT_VERSION_EXPIRATION;
+ }
+
+ @Override
+ public void valid(long creationTime) throws OMException {
+ if (noncurrentDays == null && newerNoncurrentVersions == null) {
+ throw new OMException("Invalid lifecycle configuration: "
+ + "NoncurrentVersionExpiration must specify 'NoncurrentDays', "
+ + "'NewerNoncurrentVersions', or both.",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+
+ if (noncurrentDays != null) {
+ if (noncurrentDays <= 0) {
+ throw new OMException("'NoncurrentDays' must be a positive integer "
+ + "greater than zero.", OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+
+ if (newerNoncurrentVersions != null
+ && (newerNoncurrentVersions <= 0
+ || newerNoncurrentVersions > MAX_NEWER_NONCURRENT_VERSIONS)) {
+ throw new OMException("'NewerNoncurrentVersions' must be between 1 and "
+ + MAX_NEWER_NONCURRENT_VERSIONS + ".",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+
+ @Override
+ public LifecycleAction getProtobuf() {
+ NoncurrentVersionExpiration.Builder builder =
+ NoncurrentVersionExpiration.newBuilder();
+ if (noncurrentDays != null) {
+ builder.setNoncurrentDays(noncurrentDays);
+ }
+ if (newerNoncurrentVersions != null) {
+ builder.setNewerNoncurrentVersions(newerNoncurrentVersions);
+ }
+ return LifecycleAction.newBuilder()
+ .setNoncurrentVersionExpiration(builder).build();
+ }
+
+ public static OmLCNoncurrentVersionExpiration getFromProtobuf(
+ NoncurrentVersionExpiration proto) {
+ Builder builder = new Builder();
+ if (proto.hasNoncurrentDays()) {
+ builder.setNoncurrentDays(proto.getNoncurrentDays());
+ }
+ if (proto.hasNewerNoncurrentVersions()) {
+ builder.setNewerNoncurrentVersions(proto.getNewerNoncurrentVersions());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public String toString() {
+ return "OmLCNoncurrentVersionExpiration{"
+ + "noncurrentDays=" + noncurrentDays
+ + ", newerNoncurrentVersions=" + newerNoncurrentVersions
+ + '}';
+ }
+
+ /**
+ * Builder of OmLCNoncurrentVersionExpiration.
+ */
+ public static class Builder {
+ private Integer noncurrentDays = null;
+ private Integer newerNoncurrentVersions = null;
+
+ public Builder setNoncurrentDays(int days) {
+ this.noncurrentDays = days;
+ return this;
+ }
+
+ public Builder setNewerNoncurrentVersions(int versions) {
+ this.newerNoncurrentVersions = versions;
+ return this;
+ }
+
+ public OmLCNoncurrentVersionExpiration build() {
+ return new OmLCNoncurrentVersionExpiration(this);
+ }
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
index ce9f56c30b89..32ddf55425fe 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
@@ -139,6 +139,21 @@ public OmLCAbortIncompleteMultipartUpload getAbortIncompleteMultipartUpload() {
return null;
}
+ /**
+ * Get the NoncurrentVersionExpiration action if present.
+ *
+ * @return the NoncurrentVersionExpiration action if present, null otherwise
+ */
+ @Nullable
+ public OmLCNoncurrentVersionExpiration getNoncurrentVersionExpiration() {
+ for (OmLCAction action : actions) {
+ if (action instanceof OmLCNoncurrentVersionExpiration) {
+ return (OmLCNoncurrentVersionExpiration) action;
+ }
+ }
+ return null;
+ }
+
@Nullable
public OmLCFilter getFilter() {
return filter;
@@ -187,14 +202,24 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
// Check that there is at most one Expiration action
int expirationActionCount = 0;
+ int noncurrentExpirationActionCount = 0;
for (OmLCAction action : actions) {
if (action.getActionType() == OmLCAction.ActionType.EXPIRATION) {
expirationActionCount++;
}
+ if (action.getActionType()
+ == OmLCAction.ActionType.NONCURRENT_VERSION_EXPIRATION) {
+ noncurrentExpirationActionCount++;
+ }
if (expirationActionCount > 1) {
throw new OMException("A rule can have at most one Expiration action.",
OMException.ResultCodes.INVALID_REQUEST);
}
+ if (noncurrentExpirationActionCount > 1) {
+ throw new OMException(
+ "A rule can have at most one NoncurrentVersionExpiration action.",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
action.valid(creationTime);
if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
@@ -206,6 +231,17 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
}
}
+ // A delete marker carries no tags, so a tag-filtered rule could never
+ // match one. S3 refuses the combination rather than accepting a rule that
+ // can never fire.
+ OmLCExpiration expirationAction = getExpiration();
+ if (isTagEnable && expirationAction != null
+ && expirationAction.isExpiredObjectDeleteMarker()) {
+ throw new OMException("Invalid lifecycle configuration: "
+ + "'ExpiredObjectDeleteMarker' cannot be specified with a tag "
+ + "filter.", OMException.ResultCodes.INVALID_REQUEST);
+ }
+
if (prefix != null && filter != null) {
throw new OMException("Filter and Prefix cannot be used together.",
OMException.ResultCodes.INVALID_REQUEST);
@@ -236,19 +272,24 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
* @return true is this key fits this rule and will trigger the action, otherwise false
*/
public boolean match(OmKeyInfo omKeyInfo) {
- boolean matched = false;
- // verify modification time first
- if (getExpiration().isExpired(omKeyInfo.getModificationTime())) {
- // verify prefix and filter
- if (prefix != null) {
- if (omKeyInfo.getKeyName().startsWith(prefix)) {
- matched = true;
- }
- } else {
- return filter.match(omKeyInfo);
- }
+ // verify modification time first, then prefix and filter
+ return getExpiration().isExpired(omKeyInfo.getModificationTime())
+ && matchesScope(omKeyInfo);
+ }
+
+ /**
+ * Whether the key is in this rule's scope, by prefix or by filter, without
+ * regard to any action's own condition. An action that decides expiry for
+ * itself - NoncurrentVersionExpiration, which reasons about a version's
+ * position and how long it has been noncurrent - needs the scope alone.
+ *
+ * @param omKeyInfo the key to evaluate against this rule's scope
+ */
+ public boolean matchesScope(OmKeyInfo omKeyInfo) {
+ if (prefix != null) {
+ return omKeyInfo.getKeyName().startsWith(prefix);
}
- return matched;
+ return filter.match(omKeyInfo);
}
/**
@@ -327,6 +368,10 @@ public static OmLCRule getFromProtobuf(LifecycleRule lifecycleRule, BucketLayout
builder.addAction(OmLCAbortIncompleteMultipartUpload.getFromProtobuf(
lifecycleAction.getAbortIncompleteMultipartUpload()));
}
+ if (lifecycleAction.hasNoncurrentVersionExpiration()) {
+ builder.addAction(OmLCNoncurrentVersionExpiration.getFromProtobuf(
+ lifecycleAction.getNoncurrentVersionExpiration()));
+ }
}
if (lifecycleRule.hasFilter()) {
builder.setFilter(OmLCFilter.getFromProtobuf(lifecycleRule.getFilter(), layout));
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
index 41bc5de16aa0..a476eb09961f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
@@ -34,6 +34,7 @@ public class OmLifecycleScanState {
private String lastScannedKey;
private String lastScannedDir;
private String lastScannedDirKey;
+ private String lastScannedVersionKey;
private static final Codec CODEC = new DelegatedCodec<>(
Proto2Codec.get(LifecycleScanState.getDefaultInstance()),
@@ -59,6 +60,7 @@ private OmLifecycleScanState(Builder builder) {
this.lastScannedKey = builder.lastScannedKey;
this.lastScannedDir = builder.lastScannedDir;
this.lastScannedDirKey = builder.lastScannedDirKey;
+ this.lastScannedVersionKey = builder.lastScannedVersionKey;
}
public String getBucketKey() {
@@ -105,6 +107,14 @@ public String getLastScannedDirKey() {
return lastScannedDirKey;
}
+ public String getLastScannedVersionKey() {
+ return lastScannedVersionKey;
+ }
+
+ public void setLastScannedVersionKey(String versionKey) {
+ this.lastScannedVersionKey = versionKey;
+ }
+
public LifecycleScanState getProtobuf() {
LifecycleScanState.Builder builder = LifecycleScanState.newBuilder()
.setBucketKey(bucketKey)
@@ -124,6 +134,9 @@ public LifecycleScanState getProtobuf() {
if (lastScannedDirKey != null) {
builder.setLastScannedDirKey(lastScannedDirKey);
}
+ if (lastScannedVersionKey != null) {
+ builder.setLastScannedVersionKey(lastScannedVersionKey);
+ }
return builder.build();
}
@@ -146,6 +159,9 @@ public static OmLifecycleScanState getFromProtobuf(LifecycleScanState proto) {
if (proto.hasLastScannedDirKey()) {
builder.setLastScannedDirKey(proto.getLastScannedDirKey());
}
+ if (proto.hasLastScannedVersionKey()) {
+ builder.setLastScannedVersionKey(proto.getLastScannedVersionKey());
+ }
return builder.build();
}
@@ -160,6 +176,7 @@ public String toString() {
", lastScannedKey='" + lastScannedKey + '\'' +
", lastScannedDir='" + lastScannedDir + '\'' +
", lastScannedDirKey='" + lastScannedDirKey + '\'' +
+ ", lastScannedVersionKey='" + lastScannedVersionKey + '\'' +
'}';
}
@@ -172,7 +189,8 @@ public Builder toBuilder() {
.setScanEndTime(scanEndTime)
.setLastScannedKey(lastScannedKey)
.setLastScannedDir(lastScannedDir)
- .setLastScannedDirKey(lastScannedDirKey);
+ .setLastScannedDirKey(lastScannedDirKey)
+ .setLastScannedVersionKey(lastScannedVersionKey);
}
/**
@@ -187,6 +205,7 @@ public static class Builder {
private String lastScannedKey;
private String lastScannedDir;
private String lastScannedDirKey;
+ private String lastScannedVersionKey;
public Builder setBucketKey(String bucketKey) {
this.bucketKey = bucketKey;
@@ -243,6 +262,15 @@ public String getLastScannedDirKey() {
return lastScannedDirKey;
}
+ public String getLastScannedVersionKey() {
+ return lastScannedVersionKey;
+ }
+
+ public Builder setLastScannedVersionKey(String versionKey) {
+ this.lastScannedVersionKey = versionKey;
+ return this;
+ }
+
public Builder setLastScannedDirKey(String lastScannedDirKey) {
this.lastScannedDirKey = lastScannedDirKey;
return this;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/PinnedFirstVersionIdGenerator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/PinnedFirstVersionIdGenerator.java
new file mode 100644
index 000000000000..ca8711600f93
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/PinnedFirstVersionIdGenerator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+/**
+ * Numbers versions like {@link UniqueIdVersionIdGenerator}, except that a key's
+ * first version takes {@link #FIRST_VERSION_ID}, so it can be referenced
+ * without listing the key's versions first.
+ *
+ * Known trade-off: once every version of a key has been permanently deleted,
+ * a recreated key takes the sentinel again, so an external reference to the
+ * first version resolves to the new content.
+ */
+public class PinnedFirstVersionIdGenerator extends UniqueIdVersionIdGenerator {
+
+ /**
+ * Id of a pinned first version. Smaller than any proposed id, so such a
+ * version sorts at the old end of the key's versions.
+ */
+ public static final long FIRST_VERSION_ID = 1L;
+
+ @Override
+ public long versionIdFor(long proposedVersionId, boolean hasCurrentVersion) {
+ return hasCurrentVersion ? proposedVersionId : FIRST_VERSION_ID;
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/UniqueIdVersionIdGenerator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/UniqueIdVersionIdGenerator.java
new file mode 100644
index 000000000000..fa466acdecf0
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/UniqueIdVersionIdGenerator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.hdds.utils.UniqueId;
+
+/**
+ * Numbers a version with the time it was written, through the scheme Ozone
+ * already uses for block local IDs: {@code currentTimeMillis << 16} with a
+ * 16-bit counter separating ids proposed inside one millisecond.
+ *
+ *
Needs no allocator state and no coordination. It does not promise the
+ * per-key ordering the versionedKeyTable needs — the counter can run out, and a
+ * clock can step backwards — which {@code VersionIdAllocator} establishes by
+ * taking a proposal as a floor.
+ */
+public class UniqueIdVersionIdGenerator implements VersionIdGenerator {
+
+ @Override
+ public long generateVersionId() {
+ return UniqueId.next();
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/VersionIdGenerator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/VersionIdGenerator.java
new file mode 100644
index 000000000000..f880dff26545
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/VersionIdGenerator.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Numbers an object version.
+ *
+ *
The id is proposed on the OM that received the request, in preExecute, so
+ * it travels in the replicated request and every OM applies a version that is
+ * already numbered. Nothing about it depends on the transaction carrying the
+ * write, or on OM being replicated by Ratis.
+ *
+ *
Implementations must be public with a public no-argument constructor, and
+ * are selected cluster-wide by
+ * {@link OMConfigKeys#OZONE_OM_VERSIONING_VERSION_ID_GENERATOR}.
+ */
+public interface VersionIdGenerator {
+
+ /**
+ * Unset value of the optional versionId field, carried by records written
+ * before versioning existed. Never proposed.
+ *
+ *
Not the id of the null version, which carries a proposed id like any
+ * other version and is marked by {@code isNullVersion} instead.
+ */
+ long UNSET_VERSION_ID = 0L;
+
+ /** @return the id to propose for a version being created. */
+ long generateVersionId();
+
+ /**
+ * The id a version takes when applied, given what was proposed for it. A
+ * generator that numbers some versions specially decides that here, where
+ * whether the key already has a version is settled under the write's lock.
+ *
+ * @param proposedVersionId the id the request carried
+ * @param hasCurrentVersion whether keyTable already holds a version of the key
+ */
+ default long versionIdFor(long proposedVersionId, boolean hasCurrentVersion) {
+ return proposedVersionId;
+ }
+
+ /**
+ * Instantiates the generator configured for this cluster.
+ *
+ * @throws RuntimeException if the configured class cannot be instantiated
+ */
+ static VersionIdGenerator fromConfiguration(ConfigurationSource conf) {
+ Class generatorClass = conf.getClass(
+ OMConfigKeys.OZONE_OM_VERSIONING_VERSION_ID_GENERATOR,
+ OMConfigKeys.OZONE_OM_VERSIONING_VERSION_ID_GENERATOR_DEFAULT,
+ VersionIdGenerator.class);
+ return ReflectionUtils.newInstance(generatorClass, null);
+ }
+}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
index 147255b3b573..cb24e5cbb571 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
@@ -65,6 +65,32 @@ public void testQuotaIsSetFlagsAreCorrectlySet() {
assertTrue(argsFromProto.hasQuotaInNamespace());
}
+ @Test
+ public void testVersioningStatusIsSetCorrectly() {
+ OmBucketArgs bucketArgs = OmBucketArgs.newBuilder()
+ .setBucketName("bucket")
+ .setVolumeName("volume")
+ .build();
+
+ OmBucketArgs argsFromProto = OmBucketArgs.getFromProtobuf(
+ bucketArgs.getProtobuf());
+
+ // absent means "not being changed"
+ assertNull(argsFromProto.getVersioningStatus());
+
+ bucketArgs = OmBucketArgs.newBuilder()
+ .setBucketName("bucket")
+ .setVolumeName("volume")
+ .setVersioningStatus(BucketVersioningStatus.SUSPENDED)
+ .build();
+
+ argsFromProto = OmBucketArgs.getFromProtobuf(
+ bucketArgs.getProtobuf());
+
+ assertEquals(BucketVersioningStatus.SUSPENDED,
+ argsFromProto.getVersioningStatus());
+ }
+
@Test
public void testDefaultReplicationConfigIsSetCorrectly() {
OmBucketArgs bucketArgs = OmBucketArgs.newBuilder()
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
index 3cd3d53508d3..34f3bbe6c0e9 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -54,6 +55,91 @@ public void protobufConversion() {
OmBucketInfo.getFromProtobuf(bucket.getProtobuf()));
}
+ @Test
+ public void legacyFlagDoesNotImplyAnS3VersioningStatus() {
+ // Records written before the versioningStatus field existed deserialize
+ // unchanged and keep carrying the legacy flag alone. The legacy flag
+ // selects the in-record block version list, which is a different feature
+ // from S3 versioning, so it must not be promoted to a status.
+ OzoneManagerProtocolProtos.BucketInfo oldRecord =
+ OzoneManagerProtocolProtos.BucketInfo.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket")
+ .setIsVersionEnabled(false)
+ .setStorageType(HddsProtos.StorageTypeProto.DISK)
+ .build();
+ OmBucketInfo bucket = OmBucketInfo.getFromProtobuf(oldRecord);
+ assertFalse(bucket.hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ bucket.getVersioningStatus());
+ assertFalse(bucket.getIsVersionEnabled());
+ assertEquals(bucket, OmBucketInfo.getFromProtobuf(bucket.getProtobuf()));
+
+ oldRecord = oldRecord.toBuilder().setIsVersionEnabled(true).build();
+ bucket = OmBucketInfo.getFromProtobuf(oldRecord);
+ assertFalse(bucket.hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ bucket.getVersioningStatus());
+ assertTrue(bucket.getIsVersionEnabled());
+ // the absent status survives the round trip, so a re-serialized legacy
+ // record is still distinguishable from an S3-versioned one
+ assertFalse(bucket.getProtobuf().hasVersioningStatus());
+ assertEquals(bucket, OmBucketInfo.getFromProtobuf(bucket.getProtobuf()));
+ }
+
+ @Test
+ public void versioningStatusProtobufConversion() {
+ // SUSPENDED is not representable by the legacy flag alone, so it must
+ // survive a proto round trip via the new field.
+ OmBucketInfo bucket = OmBucketInfo.newBuilder()
+ .setBucketName("bucket")
+ .setVolumeName("vol1")
+ .setVersioningStatus(BucketVersioningStatus.SUSPENDED)
+ .build();
+ assertFalse(bucket.getIsVersionEnabled());
+
+ OmBucketInfo recovered = OmBucketInfo.getFromProtobuf(bucket.getProtobuf());
+ assertEquals(BucketVersioningStatus.SUSPENDED,
+ recovered.getVersioningStatus());
+ assertFalse(recovered.getIsVersionEnabled());
+ assertEquals(bucket, recovered);
+ }
+
+ @Test
+ public void builderKeepsVersioningStatusAndLegacyFlagInSync() {
+ OmBucketInfo.Builder builder = OmBucketInfo.newBuilder()
+ .setBucketName("bucket")
+ .setVolumeName("vol1");
+
+ // no status set at all
+ assertFalse(builder.build().hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ builder.build().getVersioningStatus());
+
+ // the legacy flag sets only itself: no status is derived from it
+ builder.setIsVersionEnabled(true);
+ assertFalse(builder.build().hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ builder.build().getVersioningStatus());
+ assertTrue(builder.build().getIsVersionEnabled());
+
+ // an explicit status is authoritative and drives the legacy flag
+ builder.setVersioningStatus(BucketVersioningStatus.SUSPENDED);
+ assertTrue(builder.build().hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.SUSPENDED,
+ builder.build().getVersioningStatus());
+ assertFalse(builder.build().getIsVersionEnabled());
+
+ // ENABLED shows up as true to clients that only know the legacy flag
+ builder.setVersioningStatus(BucketVersioningStatus.ENABLED);
+ assertTrue(builder.build().getIsVersionEnabled());
+
+ // a null status is a no-op (records without the new field)
+ builder.setVersioningStatus(null);
+ assertEquals(BucketVersioningStatus.ENABLED,
+ builder.build().getVersioningStatus());
+ }
+
@Test
public void protobufConversionOfBucketLink() {
OmBucketInfo bucket = OmBucketInfo.newBuilder()
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyArgs.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyArgs.java
index 8c399ba3562c..3ae6d19537b2 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyArgs.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyArgs.java
@@ -18,7 +18,13 @@
package org.apache.hadoop.ozone.om.helpers;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -38,4 +44,84 @@ void toBuilderPreservesHeadOp(boolean headOp) {
assertEquals(headOp, subject.toBuilder().build().isHeadOp());
}
+ @Test
+ void validateAddressedVersionRejectsNamingAVersionTwice() {
+ KeyArgs keyArgs = KeyArgs.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+ .setVersionId(42L)
+ .setNullVersion(true)
+ .build();
+
+ OMException ex = assertThrows(OMException.class,
+ () -> OmKeyArgs.validateAddressedVersion(keyArgs));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = { true, false })
+ void validateAddressedVersionAcceptsEitherOnItsOwn(boolean byId)
+ throws Exception {
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1");
+ if (byId) {
+ keyArgs.setVersionId(42L);
+ } else {
+ keyArgs.setNullVersion(true);
+ }
+
+ OmKeyArgs.validateAddressedVersion(keyArgs.build());
+ }
+
+ @Test
+ void validateAddressedVersionAcceptsAddressingNoVersion() throws Exception {
+ OmKeyArgs.validateAddressedVersion(KeyArgs.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+ .build());
+ }
+
+ @Test
+ void toProtobufPreservesVersionId() {
+ OmKeyArgs subject = new OmKeyArgs.Builder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+ .setVersionId(42L)
+ .build();
+
+ assertTrue(subject.toProtobuf().hasVersionId());
+ assertEquals(42L, subject.toProtobuf().getVersionId());
+ assertFalse(subject.toProtobuf().getNullVersion());
+ }
+
+ @Test
+ void toProtobufPreservesNullVersion() {
+ OmKeyArgs subject = new OmKeyArgs.Builder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+ .setNullVersion(true)
+ .build();
+
+ assertTrue(subject.toProtobuf().getNullVersion());
+ assertFalse(subject.toProtobuf().hasVersionId());
+ }
+
+ @Test
+ void toProtobufLeavesTheVersionUnsetWhenNoneIsAddressed() {
+ OmKeyArgs subject = new OmKeyArgs.Builder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+ .build();
+
+ assertFalse(subject.toProtobuf().hasVersionId());
+ assertFalse(subject.toProtobuf().getNullVersion());
+ }
+
}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
index 285853a3a766..12846bece1c4 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
@@ -25,6 +25,7 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
@@ -116,6 +117,40 @@ public void getProtobufMessageEC() throws IOException {
assertEquals(2, config.getParity());
}
+ @Test
+ public void protobufConversionWithVersioningFields() {
+ // records without versioning fields keep them unset after a round trip
+ OmKeyInfo key = createOmKeyInfo(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
+ OzoneManagerProtocolProtos.KeyInfo proto = key.getProtobuf(ClientVersion.CURRENT_VERSION);
+ assertFalse(proto.hasVersionId());
+ assertFalse(proto.hasIsDeleteMarker());
+ assertFalse(proto.hasIsNullVersion());
+ OmKeyInfo recovered = OmKeyInfo.getFromProtobuf(proto);
+ assertNull(recovered.getVersionId());
+ assertFalse(recovered.isDeleteMarker());
+ assertFalse(recovered.isNullVersion());
+
+ // versioning fields survive a round trip and the copy constructor
+ key = createOmKeyInfo(RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+ .toBuilder()
+ .setVersionId(4242L)
+ .setDeleteMarker(true)
+ .setNullVersion(true)
+ .build();
+ recovered = OmKeyInfo.getFromProtobuf(key.getProtobuf(ClientVersion.CURRENT_VERSION));
+ assertEquals(4242L, recovered.getVersionId());
+ assertTrue(recovered.isDeleteMarker());
+ assertTrue(recovered.isNullVersion());
+
+ // records differing only in a versioning field must not compare equal
+ OmKeyInfo plain = createOmKeyInfo(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
+ assertNotEquals(plain, plain.toBuilder().setVersionId(1L).build());
+ assertNotEquals(plain, plain.toBuilder().setDeleteMarker(true).build());
+ assertNotEquals(plain, plain.toBuilder().setNullVersion(true).build());
+ }
+
private OmKeyInfo createOmKeyInfo(ReplicationConfig replicationConfig) {
return new Builder()
.setKeyName("key1")
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java
index 1aa3e2b5011c..676e02867927 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java
@@ -23,6 +23,7 @@
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
@@ -92,7 +93,7 @@ public void testCreateInValidOmLCExpiration() {
.setDate(getFutureDateString(100));
long currentTime = System.currentTimeMillis();
assertOMException(() -> exp1.build().valid(currentTime), INVALID_REQUEST,
- "Either 'days' or 'date' should be specified, but not both or neither.");
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
OmLCExpiration.Builder exp2 = new OmLCExpiration.Builder()
.setDays(-1);
@@ -102,16 +103,16 @@ public void testCreateInValidOmLCExpiration() {
OmLCExpiration.Builder exp3 = new OmLCExpiration.Builder()
.setDate(null);
assertOMException(() -> exp3.build().valid(currentTime), INVALID_REQUEST,
- "Either 'days' or 'date' should be specified, but not both or neither.");
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
OmLCExpiration.Builder exp4 = new OmLCExpiration.Builder()
.setDate("");
assertOMException(() -> exp4.build().valid(currentTime), INVALID_REQUEST,
- "Either 'days' or 'date' should be specified, but not both or neither.");
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
OmLCExpiration.Builder exp5 = new OmLCExpiration.Builder();
assertOMException(() -> exp5.build().valid(currentTime), INVALID_REQUEST,
- "Either 'days' or 'date' should be specified, but not both or neither.");
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
OmLCExpiration.Builder exp6 = new OmLCExpiration.Builder()
.setDate("10-10-2099");
@@ -172,6 +173,43 @@ public void testDateMustBeAtMidnightUTC() {
assertOMException(() -> exp5.build().valid(currentTime), INVALID_REQUEST, "'Date' must represent midnight UTC");
}
+ /**
+ * ExpiredObjectDeleteMarker removes a marker rather than expiring an object
+ * by age, so it stands alone among the three.
+ */
+ @Test
+ public void testExpiredObjectDeleteMarkerIsExclusive() {
+ long currentTime = System.currentTimeMillis();
+
+ assertDoesNotThrow(() -> new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true).build().valid(currentTime));
+
+ assertOMException(() -> new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true).setDays(30).build()
+ .valid(currentTime), INVALID_REQUEST,
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
+
+ assertOMException(() -> new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true)
+ .setDate(getFutureDateString(100)).build()
+ .valid(currentTime), INVALID_REQUEST,
+ "exactly one of 'Days', 'Date' and 'ExpiredObjectDeleteMarker'");
+ }
+
+ @Test
+ public void testExpiredObjectDeleteMarkerSurvivesProtobuf()
+ throws OMException {
+ OmLCExpiration marker = new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true).build();
+
+ OmLCExpiration back = OmLCExpiration.getFromProtobuf(
+ marker.getProtobuf().getExpiration());
+
+ assertTrue(back.isExpiredObjectDeleteMarker());
+ assertNull(back.getDays());
+ assertNull(back.getDate());
+ }
+
@Test
public void testProtobufConversion() throws OMException {
// Only Days
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCNoncurrentVersionExpiration.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCNoncurrentVersionExpiration.java
new file mode 100644
index 000000000000..024fd9b719dc
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCNoncurrentVersionExpiration.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+import static org.apache.hadoop.ozone.om.helpers.OMLCUtils.assertOMException;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests the lifecycle action that reclaims a key's noncurrent versions.
+ */
+class TestOmLCNoncurrentVersionExpiration {
+
+ private static OmLCNoncurrentVersionExpiration of(Integer days,
+ Integer versions) {
+ OmLCNoncurrentVersionExpiration.Builder builder =
+ new OmLCNoncurrentVersionExpiration.Builder();
+ if (days != null) {
+ builder.setNoncurrentDays(days);
+ }
+ if (versions != null) {
+ builder.setNewerNoncurrentVersions(versions);
+ }
+ return builder.build();
+ }
+
+ @Test
+ void eitherLimitOnItsOwnIsValid() {
+ assertDoesNotThrow(() -> of(30, null).valid(0L));
+ assertDoesNotThrow(() -> of(null, 5).valid(0L));
+ assertDoesNotThrow(() -> of(30, 5).valid(0L));
+ }
+
+ /** Unlike Expiration's Days and Date, the two limits are not exclusive. */
+ @Test
+ void aRuleThatLimitsNothingIsRejected() {
+ assertOMException(() -> of(null, null).valid(0L), INVALID_REQUEST,
+ "must specify 'NoncurrentDays', 'NewerNoncurrentVersions', or both");
+ }
+
+ @Test
+ void noncurrentDaysMustBePositive() {
+ assertOMException(() -> of(0, null).valid(0L), INVALID_REQUEST,
+ "'NoncurrentDays' must be a positive integer");
+ }
+
+ @Test
+ void newerNoncurrentVersionsIsBoundedLikeS3() {
+ assertDoesNotThrow(() -> of(null, 1).valid(0L));
+ assertDoesNotThrow(() -> of(null,
+ OmLCNoncurrentVersionExpiration.MAX_NEWER_NONCURRENT_VERSIONS)
+ .valid(0L));
+
+ assertOMException(() -> of(null, 0).valid(0L), INVALID_REQUEST,
+ "'NewerNoncurrentVersions' must be between 1 and 100");
+ assertOMException(() -> of(null,
+ OmLCNoncurrentVersionExpiration.MAX_NEWER_NONCURRENT_VERSIONS + 1)
+ .valid(0L), INVALID_REQUEST,
+ "'NewerNoncurrentVersions' must be between 1 and 100");
+ }
+
+ /**
+ * A rule that only caps how many versions are kept says nothing about age,
+ * so no version is ever expired by time under it.
+ */
+ @Test
+ void aCountOnlyRuleNeverExpiresByAge() throws Exception {
+ OmLCNoncurrentVersionExpiration rule = of(null, 5);
+ rule.valid(0L);
+
+ assertFalse(rule.isExpired(0L));
+ }
+
+ @Test
+ void aVersionExpiresOnceItHasBeenNoncurrentLongEnough() throws Exception {
+ OmLCNoncurrentVersionExpiration rule = of(1, null);
+ rule.valid(0L);
+
+ final long twoDaysAgo =
+ System.currentTimeMillis() - java.util.concurrent.TimeUnit.DAYS
+ .toMillis(2);
+ assertTrue(rule.isExpired(twoDaysAgo));
+ assertFalse(rule.isExpired(System.currentTimeMillis()));
+ }
+
+ /**
+ * The age is settled when the rule is built. Deriving it in valid() instead
+ * would make an unvalidated rule read as zero days, which expires every
+ * version the moment it becomes noncurrent.
+ */
+ @Test
+ void ageIsHonouredWithoutValidationHavingRun() {
+ OmLCNoncurrentVersionExpiration rule = of(1, null);
+
+ final long anHourAgo =
+ System.currentTimeMillis()
+ - java.util.concurrent.TimeUnit.HOURS.toMillis(1);
+ assertFalse(rule.isExpired(anHourAgo));
+ }
+
+ @Test
+ void theActionSurvivesAProtobufRoundTrip() {
+ LifecycleAction proto = of(30, 5).getProtobuf();
+ OmLCNoncurrentVersionExpiration back =
+ OmLCNoncurrentVersionExpiration.getFromProtobuf(
+ proto.getNoncurrentVersionExpiration());
+
+ assertEquals(30, back.getNoncurrentDays());
+ assertEquals(5, back.getNewerNoncurrentVersions());
+
+ OmLCNoncurrentVersionExpiration daysOnly =
+ OmLCNoncurrentVersionExpiration.getFromProtobuf(
+ of(30, null).getProtobuf().getNoncurrentVersionExpiration());
+ assertEquals(30, daysOnly.getNoncurrentDays());
+ assertNull(daysOnly.getNewerNoncurrentVersions());
+ }
+}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
index 8b39e6b7aac2..d5a4bcada3ca 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
@@ -104,6 +104,45 @@ public void testCreateInValidOmLCRule() throws OMException {
"Filter and Prefix cannot both be null.");
}
+ /**
+ * A delete marker carries no tags, so a tag-filtered rule could never match
+ * one and S3 refuses the combination.
+ */
+ @Test
+ public void testExpiredObjectDeleteMarkerRejectsATagFilter() {
+ long currentTime = System.currentTimeMillis();
+ OmLCExpiration marker = new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true).build();
+
+ OmLCRule tagged = new OmLCRule.Builder()
+ .setId("id").setEnabled(true)
+ .setFilter(getOmLCFilterBuilder(
+ null, Pair.of("key", "value"), null).build())
+ .setAction(marker)
+ .build();
+ assertOMException(() -> tagged.valid(BucketLayout.DEFAULT, currentTime),
+ INVALID_REQUEST,
+ "'ExpiredObjectDeleteMarker' cannot be specified with a tag filter");
+
+ OmLCRule taggedByAndOperator = new OmLCRule.Builder()
+ .setId("id").setEnabled(true)
+ .setFilter(getOmLCFilterBuilder(null, null, getOmLCAndOperatorBuilder(
+ "prefix", ImmutableMap.of("tag1", "value1")).build()).build())
+ .setAction(marker)
+ .build();
+ assertOMException(
+ () -> taggedByAndOperator.valid(BucketLayout.DEFAULT, currentTime),
+ INVALID_REQUEST,
+ "'ExpiredObjectDeleteMarker' cannot be specified with a tag filter");
+
+ OmLCRule byPrefix = new OmLCRule.Builder()
+ .setId("id").setEnabled(true)
+ .setPrefix("logs/")
+ .setAction(marker)
+ .build();
+ assertDoesNotThrow(() -> byPrefix.valid(BucketLayout.DEFAULT, currentTime));
+ }
+
@Test
public void testCreateFSOLCRule() throws OMException {
long currentTime = System.currentTimeMillis();
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestVersionIdGenerator.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestVersionIdGenerator.java
new file mode 100644
index 000000000000..f08c024a67e0
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestVersionIdGenerator.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.stream.Stream;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+/**
+ * Tests the generators that propose a versionId, and the configuration that
+ * selects one.
+ */
+public class TestVersionIdGenerator {
+
+ /** Every generator, so the contract below is checked against all of them. */
+ static Stream generators() {
+ return Stream.of(new UniqueIdVersionIdGenerator(),
+ new PinnedFirstVersionIdGenerator());
+ }
+
+ /**
+ * The contract every generator owes: ids come out strictly increasing, and
+ * above the reserved unset value. A clock can still break this across OMs,
+ * which is what the allocator's floor is for.
+ */
+ @ParameterizedTest
+ @MethodSource("generators")
+ void proposedIdsStrictlyIncrease(VersionIdGenerator generator) {
+ long previous = VersionIdGenerator.UNSET_VERSION_ID;
+
+ for (int i = 0; i < 10000; i++) {
+ final long versionId = generator.generateVersionId();
+ assertTrue(versionId > previous,
+ "proposed " + versionId + " after " + previous);
+ previous = versionId;
+ }
+ }
+
+ @Test
+ void theProposedIdReadsAsTheTimeOfTheWrite() {
+ final long before = System.currentTimeMillis();
+ final long versionId = new UniqueIdVersionIdGenerator().generateVersionId();
+ final long after = System.currentTimeMillis();
+
+ final long millis = versionId >> Short.SIZE;
+ assertTrue(millis >= before && millis <= after,
+ "versionId " + versionId + " does not read as a millisecond between "
+ + before + " and " + after);
+ }
+
+ /**
+ * The default generator keeps nothing of its own, so a fresh instance picks
+ * up where the last one left off and no OM has to carry allocator state.
+ */
+ @Test
+ void theDefaultGeneratorHoldsNoState() {
+ final long first = new UniqueIdVersionIdGenerator().generateVersionId();
+ final long second = new UniqueIdVersionIdGenerator().generateVersionId();
+
+ assertTrue(second > first, "expected " + second + " to come after " + first);
+ }
+
+ /** By default the applied id is simply the one that was proposed. */
+ @Test
+ void theProposalStandsUnlessAGeneratorSaysOtherwise() {
+ final VersionIdGenerator generator = new UniqueIdVersionIdGenerator();
+
+ assertEquals(5000L, generator.versionIdFor(5000L, false));
+ assertEquals(5000L, generator.versionIdFor(5000L, true));
+ }
+
+ /** Only a key's first version is special; later ones are numbered normally. */
+ @Test
+ void onlyTheFirstVersionIsPinned() {
+ final VersionIdGenerator generator = new PinnedFirstVersionIdGenerator();
+
+ assertEquals(PinnedFirstVersionIdGenerator.FIRST_VERSION_ID,
+ generator.versionIdFor(5000L, false));
+ assertEquals(5000L, generator.versionIdFor(5000L, true));
+ }
+
+ /**
+ * versionedKeyTable orders versions by Long.MAX_VALUE - versionId, so the
+ * sentinel has to stay below every proposed id to sort at the old end of the
+ * key.
+ */
+ @Test
+ void theSentinelSortsBeforeEveryProposedId() {
+ final long sentinel = PinnedFirstVersionIdGenerator.FIRST_VERSION_ID;
+ final long proposed =
+ new PinnedFirstVersionIdGenerator().generateVersionId();
+
+ assertTrue(sentinel < proposed,
+ "sentinel " + sentinel + " does not sort before " + proposed);
+ assertTrue(Long.MAX_VALUE - sentinel > Long.MAX_VALUE - proposed,
+ "the sentinel does not sort at the old end of the key");
+ }
+
+ /**
+ * The null version is marked by isNullVersion and carries a proposed id like
+ * any other version, so nothing of it is reserved; the sentinel only has to
+ * stay clear of the unset value a pre-versioning record carries.
+ */
+ @Test
+ void theSentinelDoesNotCollideWithTheNullSlot() {
+ assertTrue(PinnedFirstVersionIdGenerator.FIRST_VERSION_ID
+ > VersionIdGenerator.UNSET_VERSION_ID);
+ }
+
+ @Test
+ void timeBasedIdsAreTheClusterDefault() {
+ assertInstanceOf(UniqueIdVersionIdGenerator.class,
+ VersionIdGenerator.fromConfiguration(new OzoneConfiguration()));
+ }
+
+ @Test
+ void theGeneratorIsSelectedByClassName() {
+ assertInstanceOf(FixedVersionIdGenerator.class,
+ VersionIdGenerator.fromConfiguration(
+ configuredWith(FixedVersionIdGenerator.class.getName())));
+ }
+
+ @Test
+ void thePinnedGeneratorIsSelectedByClassName() {
+ assertInstanceOf(PinnedFirstVersionIdGenerator.class,
+ VersionIdGenerator.fromConfiguration(configuredWith(
+ PinnedFirstVersionIdGenerator.class.getName())));
+ }
+
+ @Test
+ void anUnknownGeneratorClassIsRejected() {
+ assertThrows(RuntimeException.class,
+ () -> VersionIdGenerator.fromConfiguration(
+ configuredWith("org.apache.hadoop.ozone.om.helpers.NoSuchThing")));
+ }
+
+ @Test
+ void aClassThatIsNotAGeneratorIsRejected() {
+ assertThrows(RuntimeException.class,
+ () -> VersionIdGenerator.fromConfiguration(
+ configuredWith(String.class.getName())));
+ }
+
+ private static OzoneConfiguration configuredWith(String generatorClass) {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_VERSIONING_VERSION_ID_GENERATOR,
+ generatorClass);
+ return conf;
+ }
+
+ /** A generator that exists only to be named in configuration. */
+ public static class FixedVersionIdGenerator implements VersionIdGenerator {
+ @Override
+ public long generateVersionId() {
+ return 1L;
+ }
+ }
+}
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 25de443296c8..dc34cc4bf266 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -166,6 +166,8 @@ enum Type {
GetLifecycleServiceStatus = 150;
SetLifecycleServiceStatus = 151;
SaveLifecycleScanState = 152;
+ // Submitted by the lifecycle scan to reclaim noncurrent object versions.
+ ReclaimObjectVersions = 153;
}
enum SafeMode {
@@ -326,6 +328,7 @@ message OMRequest {
optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 151;
optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 152;
optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 153;
+ optional ReclaimObjectVersionsRequest reclaimObjectVersionsRequest = 154;
}
message OMResponse {
@@ -471,6 +474,7 @@ message OMResponse {
optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 150;
optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 151;
optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 152;
+ optional ReclaimObjectVersionsResponse reclaimObjectVersionsResponse = 153;
}
enum Status {
@@ -609,6 +613,12 @@ enum Status {
LIFECYCLE_CONFIGURATION_NOT_FOUND = 102;
UPDATE_ID_NOT_MATCH = 103;
+
+ // The addressed version exists but is a delete marker. Kept distinct from
+ // KEY_NOT_FOUND so that the S3 Gateway can answer such a read with 405
+ // MethodNotAllowed rather than 404; the gateway mapping is added with the
+ // rest of the endpoints.
+ KEY_IS_DELETE_MARKER = 104;
}
/**
@@ -818,6 +828,12 @@ message BucketInfo {
optional uint64 snapshotUsedBytes = 21;
optional uint64 snapshotUsedNamespace = 22;
repeated hadoop.hdds.KeyValue tags = 23;
+ // S3-compatible object versioning status. When absent the bucket carries
+ // no S3 versioning status at all: it reads as UNVERSIONED whatever
+ // isVersionEnabled says, and that flag alone drives the legacy in-record
+ // block version list. When set it is authoritative, and isVersionEnabled
+ // is kept in sync with it (VERSIONING_ENABLED -> true).
+ optional BucketVersioningStatusProto versioningStatus = 24;
}
enum BucketLayoutProto {
@@ -826,6 +842,17 @@ enum BucketLayoutProto {
OBJECT_STORE = 3;
}
+/**
+ * S3-compatible bucket versioning state machine:
+ * UNVERSIONED -> VERSIONING_ENABLED <-> VERSIONING_SUSPENDED.
+ * Once enabled, a bucket can never return to UNVERSIONED.
+ */
+enum BucketVersioningStatusProto {
+ UNVERSIONED = 1;
+ VERSIONING_ENABLED = 2;
+ VERSIONING_SUSPENDED = 3;
+}
+
/**
* Cipher suite.
*/
@@ -892,6 +919,9 @@ message BucketArgs {
optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 11;
optional BucketEncryptionInfoProto bekInfo = 12;
repeated hadoop.hdds.KeyValue tags = 13;
+ // S3-compatible object versioning status. Takes precedence over
+ // isVersionEnabled when both are set.
+ optional BucketVersioningStatusProto versioningStatus = 14;
}
message PrefixInfo {
@@ -1133,6 +1163,23 @@ message KeyArgs {
// the given ETag for the operation to succeed. This is used for
// S3 conditional writes with the If-Match header.
optional string expectedETag = 24;
+
+ // S3-compatible object versioning: the versionId proposed for the version
+ // this request creates. It is generated in preExecute, on the OM that
+ // received the request, so every OM applies the same value; the applying
+ // OM raises it to come after the key's current version if the proposal does
+ // not already. Set on every request that can create a version, and ignored
+ // unless the bucket has versioning enabled when the request is applied.
+ optional uint64 proposedVersionId = 25;
+
+ // S3-compatible object versioning: addresses one specific version of the key
+ // instead of its current version. Distinct from proposedVersionId above,
+ // which numbers a version being created rather than selecting an existing
+ // one. nullVersion selects the key's null version slot, which cannot be
+ // addressed by id because a null version carries a normally generated
+ // versionId like any other version. At most one of the two may be set.
+ optional uint64 versionId = 26;
+ optional bool nullVersion = 27;
}
message KeyLocation {
@@ -1226,6 +1273,22 @@ message KeyInfo {
// This allows a key to be created an committed atomically if the original has not
// been modified.
optional uint64 expectedDataGeneration = 22;
+ // S3-compatible object versioning fields. versionId identifies an object
+ // version: allocated once when the version is written and then frozen,
+ // and increasing within a key so its versions can be ordered newest
+ // first. A delete marker is a record with isDeleteMarker set and no data
+ // blocks. isNullVersion marks the single overwritable "null version" slot
+ // per key (writes while versioning is suspended, or objects that predate
+ // enabling versioning).
+ optional uint64 versionId = 23;
+ optional bool isDeleteMarker = 24;
+ optional bool isNullVersion = 25;
+
+ // When this record stopped being the key's current version, which is when
+ // the version that superseded it was written. Set on the record as it is
+ // moved into the versionedKeyTable and unset while it is current. This is
+ // the clock a NoncurrentVersionExpiration rule counts from.
+ optional uint64 noncurrentTime = 26;
}
// KeyInfoProtoLight is a lightweight subset of KeyInfo message containing
@@ -1446,12 +1509,32 @@ message DeleteKeyArgs {
required string bucketName = 2;
repeated string keys = 3;
repeated uint64 updateIDs = 4; // each key's update ID when key is identified for deletion
+ // S3-compatible object versioning: the versionId proposed for the delete
+ // markers this request creates. One is enough for the whole batch, because
+ // a versionId only has to increase within a single key; the applying OM
+ // raises it per key against that key's current version.
+ optional uint64 proposedVersionId = 5;
+ // Entries of an S3 DeleteObjects that name a version: each permanently
+ // deletes that version, as a DeleteKey with a versionId does.
+ // A key may also be in keys: its delete marker is put first, then the
+ // named version is deleted.
+ repeated KeyVersion keyVersions = 6;
+}
+
+// One version of a key, named by versionId or as the key's null version.
+message KeyVersion {
+ required string key = 1;
+ optional uint64 versionId = 2;
+ optional bool nullVersion = 3;
}
message DeleteKeyError {
optional string key = 1;
optional string errorCode = 2;
optional string errorMsg = 3;
+ // set when the entry that failed named a version
+ optional uint64 versionId = 4;
+ optional bool nullVersion = 5;
}
message DeleteKeysResponse {
@@ -2569,12 +2652,26 @@ message LifecycleFilter {
message LifecycleExpiration {
optional uint32 days = 1;
optional string date = 2;
+ // On a bucket with S3-compatible versioning, removes a delete marker once it
+ // is the key's only remaining version, taking the key with it. Exclusive with
+ // days and date: the action removes a marker rather than expiring an object.
+ optional bool expiredObjectDeleteMarker = 3;
}
message AbortIncompleteMultipartUpload {
optional uint32 daysAfterInitiation = 1;
}
+// Reclaims the noncurrent versions of a key on a bucket with S3-compatible
+// versioning. At least one of the two must be set; when both are, a version is
+// reclaimed once either says so.
+message NoncurrentVersionExpiration {
+ // days a version is retained after it stopped being current
+ optional uint32 noncurrentDays = 1;
+ // how many noncurrent versions of a key are kept, newest first
+ optional uint32 newerNoncurrentVersions = 2;
+}
+
message LifecycleRule {
required string id = 1;
required bool enabled = 2;
@@ -2586,6 +2683,7 @@ message LifecycleRule {
message LifecycleAction {
optional LifecycleExpiration expiration = 1;
optional AbortIncompleteMultipartUpload abortIncompleteMultipartUpload = 2;
+ optional NoncurrentVersionExpiration noncurrentVersionExpiration = 3;
}
message LifecycleConfiguration {
@@ -2650,6 +2748,38 @@ message SetLifecycleServiceStatusRequest {
message SetLifecycleServiceStatusResponse {
}
+/**
+ * Noncurrent object versions of one bucket that the lifecycle scan selected for
+ * reclamation, addressed by their versionedKeyTable dbKeys.
+ */
+// One record the scan selected, named by its dbKey and by the write that
+// produced it. A dbKey names a position rather than a record: a marker key is
+// the plain key name, and holds whatever version of the key is current. Scan
+// and apply are separated by replication, so the record can be replaced in
+// between, and the updateId is what tells the record the rules expired apart
+// from whatever took its place.
+message ObjectVersionRecord {
+ required string dbKey = 1;
+ required uint64 updateId = 2;
+}
+
+message ObjectVersionsBucket {
+ required string volumeName = 1;
+ required string bucketName = 2;
+ repeated ObjectVersionRecord versions = 3;
+ // keyTable records of keys whose only remaining version is a delete marker.
+ // These leave the keyTable rather than the versionedKeyTable, so the key
+ // disappears entirely.
+ repeated ObjectVersionRecord markers = 4;
+}
+
+message ReclaimObjectVersionsRequest {
+ repeated ObjectVersionsBucket versionsPerBucket = 1;
+}
+
+message ReclaimObjectVersionsResponse {
+}
+
message LifecycleScanState {
optional string bucketKey = 1; // volume/bucket
optional uint64 bucketObjID = 2;
@@ -2660,6 +2790,9 @@ message LifecycleScanState {
optional string lastScannedDir = 7;
optional string lastScannedDirKey = 8;
optional string lastScannedMpuKey = 9;
+ // Where the versionedKeyTable scan left off. Its own field because that scan
+ // walks a different table than lastScannedKey's.
+ optional string lastScannedVersionKey = 10;
}
message SaveLifecycleScanStateRequest {
diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index d29db11f2ee0..e357a5eb8b1d 100644
--- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -175,6 +175,24 @@ public interface OMMetadataManager extends DBStoreHAManager, AutoCloseable {
*/
String getOzoneKey(String volume, String bucket, String key);
+ /**
+ * Given a volume, bucket, key and versionId, return the corresponding
+ * versionedKeyTable DB key: the versionId is appended as fixed-width hex of
+ * (Long.MAX_VALUE - versionId), so all versions of a key are adjacent and
+ * ordered newest first.
+ */
+ String getVersionedOzoneKey(String volume, String bucket, String key, long versionId);
+
+ /**
+ * Prefix under which all noncurrent versions of the given key are stored in
+ * the versionedKeyTable. The key name is separated from the versionId suffix
+ * by OM_VERSIONED_KEY_SEPARATOR rather than OM_KEY_PREFIX, so that a key's
+ * versions stay contiguous under this prefix and sort before any key nested
+ * under it: seeking this prefix yields exactly that key's versions, newest
+ * first.
+ */
+ String getVersionedOzoneKeyPrefix(String volume, String bucket, String key);
+
/**
* Get DB key for a key or prefix in an FSO bucket given existing
* volume and bucket names.
@@ -408,6 +426,14 @@ List getExpiredMultipartUploads(
Table getKeyTable(BucketLayout bucketLayout);
+ /**
+ * Returns the versionedKeyTable holding noncurrent object versions
+ * (including noncurrent delete markers) of versioning-enabled buckets.
+ *
+ * @return versionedKeyTable.
+ */
+ Table getVersionedKeyTable();
+
/**
* Returns the FileTable.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
index 3fbceeeae9c7..8f9de535323c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
@@ -30,7 +30,8 @@ public enum OMSystemAction implements AuditAction {
KEY_DELETION,
SNAPSHOT_MOVE_TABLE_KEYS,
SNAPSHOT_PURGE,
- SNAPSHOT_SET_PROPERTY;
+ SNAPSHOT_SET_PROPERTY,
+ OBJECT_VERSION_CLEANUP;
@Override
public String getAction() {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
index 56ccdac79b6d..29756763c558 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
@@ -138,6 +138,17 @@ public final class DeletingServiceMetrics {
@Metric("Last Purge Key transactionId on Active Object Store")
private MutableGaugeLong lastAOSPurgeTransactionId;
+ /*
+ * Object version reclamation metrics. The submitted and reclaimed counts
+ * differ by the versions that were permanently deleted or promoted between
+ * the lifecycle scan selecting them and the request being applied.
+ */
+ @Metric("Total no. of object versions sent for reclamation")
+ private MutableGaugeLong numObjectVersionsSentForReclaim;
+
+ @Metric("Total no. of object versions reclaimed")
+ private MutableGaugeLong numObjectVersionsReclaimed;
+
private DeletingServiceMetrics() {
this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
}
@@ -348,6 +359,22 @@ public synchronized void setLastAOSTransactionInfo(TransactionInfo transactionIn
}
}
+ public void incrNumObjectVersionsSentForReclaim(long versions) {
+ this.numObjectVersionsSentForReclaim.incr(versions);
+ }
+
+ public MutableGaugeLong getNumObjectVersionsSentForReclaim() {
+ return numObjectVersionsSentForReclaim;
+ }
+
+ public MutableGaugeLong getNumObjectVersionsReclaimed() {
+ return numObjectVersionsReclaimed;
+ }
+
+ public void incrNumObjectVersionsReclaimed(long versions) {
+ this.numObjectVersionsReclaimed.incr(versions);
+ }
+
@VisibleForTesting
public void resetDirectoryMetrics() {
numDirsPurged.set(0);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 40ca7c3002ac..8a50715fe187 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -622,16 +622,38 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args, BucketLayout bucketLayout)
.validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(), keyName,
bucketLayout);
+ // Only OBJECT_STORE buckets can hold versions, so addressing one
+ // anywhere else is refused rather than answered as a missing key: a
+ // LEGACY bucket reaches the same lookup path as OBJECT_STORE and would
+ // otherwise scan the versionedKeyTable and report the key not found.
+ if (args.addressesVersion()
+ && bucketLayout != BucketLayout.OBJECT_STORE) {
+ throw new OMException("Object versioning is only supported on "
+ + BucketLayout.OBJECT_STORE + " buckets",
+ ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+
if (bucketLayout.isFileSystemOptimized()) {
value = getOmKeyInfoFSO(volumeName, bucketName, keyName);
} else {
value = getOmKeyInfo(volumeName, bucketName, keyName, bucketLayout);
+ if (args.addressesVersion()) {
+ value = getAddressedVersion(args, volumeName, bucketName, keyName,
+ value);
+ }
if (value != null) {
// For Legacy & OBS buckets, any key is a file by default. This is to
// keep getKeyInfo compatible with OFS clients.
value.setFile(true);
}
}
+ if (value != null && value.isDeleteMarker()) {
+ // Addressing a delete marker by version is answered with 405 by S3,
+ // while a request that lands on a current marker is a plain 404.
+ throw new OMException("Key: " + keyName + " is a delete marker",
+ args.addressesVersion()
+ ? ResultCodes.KEY_IS_DELETE_MARKER : ResultCodes.KEY_NOT_FOUND);
+ }
} catch (IOException ex) {
if (ex instanceof OMException) {
throw ex;
@@ -697,6 +719,38 @@ private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName,
.get(keyBytes);
}
+ /**
+ * Resolves the version addressed by {@code args} for a key whose current
+ * version is {@code current}. The current version is checked first, so a
+ * request naming the current version costs no extra read; otherwise the
+ * version is looked up in the versionedKeyTable.
+ *
+ * @param current the key's current version, or null when the key has none
+ * @return the addressed version, or null when it does not exist
+ */
+ private OmKeyInfo getAddressedVersion(OmKeyArgs args, String volumeName,
+ String bucketName, String keyName, OmKeyInfo current) throws IOException {
+ if (args.isNullVersion()) {
+ if (current != null && current.isNullVersionRecord()) {
+ return current;
+ }
+ // The null version carries a normally generated versionId, so no dbKey
+ // addresses it: it has to be searched for by attribute, in the cache as
+ // well as in the DB.
+ Pair nullVersion = NoncurrentVersions.nullVersion(
+ metadataManager, volumeName, bucketName, keyName);
+ return nullVersion == null ? null : nullVersion.getValue();
+ }
+
+ long versionId = args.getVersionId();
+ if (current != null && current.getVersionId() != null
+ && current.getVersionId() == versionId) {
+ return current;
+ }
+ return metadataManager.getVersionedKeyTable().get(metadataManager
+ .getVersionedOzoneKey(volumeName, bucketName, keyName, versionId));
+ }
+
/**
* Look up will return only closed fileInfo. This will return null if the
* keyName is a directory or if the keyName is still open for writing.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/NoncurrentVersions.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/NoncurrentVersions.java
new file mode 100644
index 000000000000..363cf8d6eff6
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/NoncurrentVersions.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Searches the noncurrent versions of a single key in the versionedKeyTable.
+ *
+ * A point lookup by versionId is a plain {@link Table#get}, which consults
+ * the cache; the searches here are for versions addressed by an attribute
+ * rather than by id, which no key of the table encodes.
+ */
+public final class NoncurrentVersions {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NoncurrentVersions.class);
+
+ private NoncurrentVersions() {
+ }
+
+ /**
+ * Whether the key still has any noncurrent version.
+ *
+ *
Not a plain prefix iteration: {@link Table#iterator} reads RocksDB
+ * alone, so a version demoted by a transaction the double buffer has not
+ * flushed yet is invisible to it. Answering "none left" then removes the
+ * key's delete marker and promotes that version back to current,
+ * resurrecting an object the user deleted.
+ */
+ public static boolean any(OMMetadataManager omMetadataManager,
+ String volumeName, String bucketName, String keyName)
+ throws IOException {
+ return newestMatching(omMetadataManager, volumeName, bucketName, keyName,
+ keyInfo -> true) != null;
+ }
+
+ /**
+ * Returns the key's null version as a (dbKey, keyInfo) pair, or null if the
+ * key has no noncurrent null version. A key has at most one.
+ */
+ public static Pair nullVersion(
+ OMMetadataManager omMetadataManager, String volumeName,
+ String bucketName, String keyName) throws IOException {
+ return nullVersion(null, omMetadataManager, volumeName, bucketName,
+ keyName);
+ }
+
+ /** As above, reporting the lookup's cost through {@code ozoneManager}. */
+ public static Pair nullVersion(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager, String volumeName,
+ String bucketName, String keyName) throws IOException {
+ return newestMatching(ozoneManager, omMetadataManager, volumeName,
+ bucketName, keyName, OmKeyInfo::isNullVersionRecord);
+ }
+
+ /**
+ * Returns the newest noncurrent version of the key that satisfies
+ * {@code filter}, or null if there is none. Versions of a key are adjacent
+ * and ordered newest first, so the smallest matching dbKey wins.
+ *
+ * Both the table cache and the DB are searched, and neither can stand in
+ * for the other:
+ *
+ *
+ * - {@link Table#iterator} reads RocksDB directly and does not consult
+ * the cache, so a version written by a transaction that the double
+ * buffer has not flushed yet is invisible to it, and a version removed
+ * by such a transaction still appears in it;
+ * - the cache only holds the current flush window, so every older
+ * version of the key exists in the DB only.
+ *
+ *
+ * The search does not stop at the first cache match either. Versions are
+ * demoted into the versionedKeyTable in increasing versionId order, so in
+ * practice a cached version is newer than every flushed one, but that is a
+ * property of the write paths rather than something enforced here, and
+ * relying on it would silently promote the wrong version if a later write
+ * path broke it. The DB search costs one seek, since it stops at the first
+ * match.
+ */
+ public static Pair newestMatching(
+ OMMetadataManager omMetadataManager, String volumeName,
+ String bucketName, String keyName, Predicate filter)
+ throws IOException {
+ return newestMatching(null, omMetadataManager, volumeName, bucketName,
+ keyName, filter);
+ }
+
+ /**
+ * As above, timed. The lookup walks the versionedKeyTable cache in full and
+ * opens an iterator on the DB, both of which a write path pays for inline,
+ * so it is measured and a slow one is named: a RocksDB iterator can be
+ * expensive to create under load, and this is where that would show.
+ *
+ * @param ozoneManager the OM to report through, or null not to report
+ */
+ public static Pair newestMatching(
+ OzoneManager ozoneManager, OMMetadataManager omMetadataManager,
+ String volumeName, String bucketName, String keyName,
+ Predicate filter) throws IOException {
+ if (ozoneManager == null) {
+ return lookup(omMetadataManager, volumeName, bucketName, keyName, filter);
+ }
+
+ final long startNs = System.nanoTime();
+ try {
+ return lookup(omMetadataManager, volumeName, bucketName, keyName, filter);
+ } finally {
+ final long elapsedNs = System.nanoTime() - startNs;
+ if (ozoneManager.getPerfMetrics() != null) {
+ ozoneManager.getPerfMetrics()
+ .addNoncurrentVersionLookupLatency(elapsedNs);
+ }
+ final long thresholdMs =
+ ozoneManager.getVersioningLookupWarningThresholdMs();
+ if (TimeUnit.NANOSECONDS.toMillis(elapsedNs) > thresholdMs) {
+ LOG.warn("Looking up the noncurrent versions of {}/{}/{} took {} ms, "
+ + "over the {} ms threshold.", volumeName, bucketName, keyName,
+ TimeUnit.NANOSECONDS.toMillis(elapsedNs), thresholdMs);
+ }
+ }
+ }
+
+ private static Pair lookup(
+ OMMetadataManager omMetadataManager, String volumeName,
+ String bucketName, String keyName, Predicate filter)
+ throws IOException {
+ final String prefix = omMetadataManager.getVersionedOzoneKeyPrefix(
+ volumeName, bucketName, keyName);
+ final Table table =
+ omMetadataManager.getVersionedKeyTable();
+
+ String bestKey = null;
+ OmKeyInfo bestValue = null;
+
+ // Entries of transactions that are not flushed yet. The cache is not
+ // sorted, so it has to be scanned; it only holds this table's writes from
+ // the current flush window.
+ Iterator, CacheValue>> cacheIterator =
+ table.cacheIterator();
+ while (cacheIterator.hasNext()) {
+ Map.Entry, CacheValue> entry = cacheIterator.next();
+ String dbKey = entry.getKey().getCacheKey();
+ OmKeyInfo value = entry.getValue().getCacheValue();
+ // a null value is a tombstone: the version is deleted but not flushed
+ if (value == null || !dbKey.startsWith(prefix) || !filter.test(value)) {
+ continue;
+ }
+ if (bestKey == null || dbKey.compareTo(bestKey) < 0) {
+ bestKey = dbKey;
+ bestValue = value;
+ }
+ }
+
+ try (Table.KeyValueIterator versions = table.iterator(prefix)) {
+ while (versions.hasNext()) {
+ Table.KeyValue entry = versions.next();
+ String dbKey = entry.getKey();
+ CacheValue cached = table.getCacheValue(new CacheKey<>(dbKey));
+ if (cached != null && cached.getCacheValue() == null) {
+ continue;
+ }
+ OmKeyInfo value = cached != null ? cached.getCacheValue() : entry.getValue();
+ if (!filter.test(value)) {
+ continue;
+ }
+ if (bestKey == null || dbKey.compareTo(bestKey) < 0) {
+ bestKey = dbKey;
+ bestValue = value;
+ }
+ // DB entries ascend, so the first match is the smallest one in the DB
+ break;
+ }
+ }
+ return bestKey == null ? null : Pair.of(bestKey, bestValue);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
index 3f965ba025e6..6ac71ef82278 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
@@ -181,6 +181,10 @@ public class OMPerformanceMetrics {
@Metric(about = "creteKeySuccess latency in nanoseconds")
private MutableRate createKeySuccessLatencyNs;
+ @Metric(about = "Looking up a key's noncurrent versions on the write path, "
+ + "in nanoseconds")
+ private MutableRate noncurrentVersionLookupLatencyNs;
+
public static OMPerformanceMetrics register() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME,
@@ -193,6 +197,10 @@ public static void unregister() {
ms.unregisterSource(SOURCE_NAME);
}
+ public void addNoncurrentVersionLookupLatency(long latencyInNs) {
+ noncurrentVersionLookupLatencyNs.add(latencyInNs);
+ }
+
public void addLookupLatency(long latencyInNs) {
lookupLatencyNs.add(latencyInNs);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 99b1c18d8ec2..5c795fca0a16 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -22,6 +22,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_VERSIONED_KEY_SEPARATOR;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES;
@@ -39,6 +40,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_KEY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_RENAMED_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VOLUME_TABLE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
@@ -162,6 +164,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
private Table volumeTable;
private Table bucketTable;
private Table keyTable;
+ private Table versionedKeyTable;
private Table openKeyTable;
private Table multipartInfoTable;
@@ -381,6 +384,11 @@ public Table getKeyTable(BucketLayout bucketLayout) {
return keyTable;
}
+ @Override
+ public Table getVersionedKeyTable() {
+ return versionedKeyTable;
+ }
+
@Override
public Table getFileTable() {
return fileTable;
@@ -499,6 +507,7 @@ protected void initializeOmTables(CacheType cacheType,
volumeTable = initializer.get(OMDBDefinition.VOLUME_TABLE_DEF, cacheType);
bucketTable = initializer.get(OMDBDefinition.BUCKET_TABLE_DEF, cacheType);
keyTable = initializer.get(OMDBDefinition.KEY_TABLE_DEF);
+ versionedKeyTable = initializer.get(OMDBDefinition.VERSIONED_KEY_TABLE_DEF);
openKeyTable = initializer.get(OMDBDefinition.OPEN_KEY_TABLE_DEF);
multipartInfoTable = initializer.get(OMDBDefinition.MULTIPART_INFO_TABLE_DEF);
@@ -657,6 +666,17 @@ public String getOzoneKey(String volume, String bucket, String key) {
return builder.toString();
}
+ @Override
+ public String getVersionedOzoneKey(String volume, String bucket, String key, long versionId) {
+ return getVersionedOzoneKeyPrefix(volume, bucket, key)
+ + String.format("%016x", Long.MAX_VALUE - versionId);
+ }
+
+ @Override
+ public String getVersionedOzoneKeyPrefix(String volume, String bucket, String key) {
+ return getOzoneKey(volume, bucket, key) + OM_VERSIONED_KEY_SEPARATOR;
+ }
+
@Override
public String getOzoneKeyFSO(String volumeName,
String bucketName,
@@ -2002,13 +2022,14 @@ public boolean containsIncompleteMPUs(String volume, String bucket)
public TablePrefixInfo getTableBucketPrefix(String volume, String bucket) throws IOException {
String keyPrefix = getBucketKeyPrefix(volume, bucket);
String keyPrefixFso = getBucketKeyPrefixFSO(volume, bucket);
- // Set value to 12 to avoid creating too big a HashTable unnecessarily.
- Map tablePrefixMap = new HashMap<>(12, 1.0f);
+ // Set value to 13 to avoid creating too big a HashTable unnecessarily.
+ Map tablePrefixMap = new HashMap<>(13, 1.0f);
tablePrefixMap.put(VOLUME_TABLE, getVolumeKey(volume));
tablePrefixMap.put(BUCKET_TABLE, getBucketKey(volume, bucket));
tablePrefixMap.put(KEY_TABLE, keyPrefix);
+ tablePrefixMap.put(VERSIONED_KEY_TABLE, keyPrefix);
tablePrefixMap.put(DELETED_TABLE, keyPrefix);
tablePrefixMap.put(SNAPSHOT_RENAMED_TABLE, keyPrefix);
tablePrefixMap.put(OPEN_KEY_TABLE, keyPrefix);
@@ -2031,6 +2052,7 @@ public String getTableBucketPrefix(String tableName, String volume, String bucke
case BUCKET_TABLE:
return getBucketKey(volume, bucket);
case KEY_TABLE:
+ case VERSIONED_KEY_TABLE:
case DELETED_TABLE:
case SNAPSHOT_RENAMED_TABLE:
case OPEN_KEY_TABLE:
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 09381bacb3d8..662d444fdb03 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -87,6 +87,8 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_READ_THREADPOOL_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GRPC_SERVER_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT;
@@ -424,6 +426,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private BucketManager bucketManager;
private KeyManager keyManager;
private PrefixManagerImpl prefixManager;
+ private final VersionIdAllocator versionIdAllocator;
+ /**
+ * Parsed once rather than per call: the noncurrent version lookup consults
+ * it on the write path, and reading it from the configuration each time
+ * costs a variable substitution and a parse.
+ */
+ private final long versioningLookupWarningThresholdMs;
private final UpgradeFinalizer upgradeFinalizer;
private ExecutorService edekCacheLoader = null;
@@ -511,6 +520,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private final BucketUtilizationMetrics bucketUtilizationMetrics;
private boolean fsSnapshotEnabled;
+ private boolean snapshotVersioningCoexistenceAllowed;
private String omHostName;
@@ -576,6 +586,11 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion());
upgradeFinalizer = new OMUpgradeFinalizer(versionManager);
+ versionIdAllocator = new VersionIdAllocator(conf);
+ versioningLookupWarningThresholdMs = conf.getTimeDuration(
+ OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD,
+ OZONE_OM_VERSIONING_LOOKUP_WARNING_THRESHOLD_DEFAULT,
+ TimeUnit.MILLISECONDS);
replicationConfigValidator =
conf.getObject(ReplicationConfigValidator.class);
@@ -1047,6 +1062,17 @@ public void close() {
fsSnapshotEnabled = configuration.getBoolean(
OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY,
OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_DEFAULT);
+ snapshotVersioningCoexistenceAllowed = configuration.getBoolean(
+ OMConfigKeys.OZONE_OM_SNAPSHOT_VERSIONING_COEXISTENCE_ENABLED,
+ OMConfigKeys.OZONE_OM_SNAPSHOT_VERSIONING_COEXISTENCE_ENABLED_DEFAULT);
+ if (snapshotVersioningCoexistenceAllowed) {
+ LOG.warn("{} is enabled. Snapshots and S3 object versioning may now be "
+ + "used on the same bucket, which is UNSAFE: version reclamation is "
+ + "not snapshot-aware yet, so reclaiming a noncurrent version can "
+ + "delete blocks a snapshot still references. This setting exists "
+ + "for testing only and must not be used on a production cluster.",
+ OMConfigKeys.OZONE_OM_SNAPSHOT_VERSIONING_COEXISTENCE_ENABLED);
+ }
omSnapshotManager = new OmSnapshotManager(this);
// Snapshot metrics
@@ -2406,6 +2432,23 @@ public long getObjectIdFromTxId(long trxnId) {
trxnId);
}
+ /**
+ * Assigns the versionId of a version being committed, using the
+ * {@link org.apache.hadoop.ozone.om.helpers.VersionIdGenerator} configured
+ * for this cluster.
+ */
+ /**
+ * How long a noncurrent version lookup may take before it is worth logging,
+ * in milliseconds.
+ */
+ public long getVersioningLookupWarningThresholdMs() {
+ return versioningLookupWarningThresholdMs;
+ }
+
+ public VersionIdAllocator getVersionIdAllocator() {
+ return versionIdAllocator;
+ }
+
/**
*
* @return Gets the stored layout version from the DB meta table.
@@ -4827,6 +4870,19 @@ public boolean isFilesystemSnapshotEnabled() {
return fsSnapshotEnabled;
}
+ /**
+ * Whether snapshots and S3 object versioning may be used on the same bucket.
+ * Unsafe and false by default: version reclamation is not snapshot-aware
+ * yet, so reclaiming a noncurrent version can delete blocks a snapshot still
+ * references. It exists so that the integration tests for that work can be
+ * written against a real mixed-state bucket.
+ *
+ * @return true if the exclusion checks are lifted, false otherwise.
+ */
+ public boolean isSnapshotVersioningCoexistenceAllowed() {
+ return snapshotVersioningCoexistenceAllowed;
+ }
+
/**
* Get DB updates since a specific sequence number.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VersionIdAllocator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VersionIdAllocator.java
new file mode 100644
index 000000000000..e137059788e5
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VersionIdAllocator.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
+
+/**
+ * Settles the versionId of a version as it is applied, from the id proposed for
+ * it before the request was replicated.
+ *
+ * The versionedKeyTable orders a key's versions by
+ * {@code Long.MAX_VALUE - versionId}, so the ids of one key have to increase in
+ * the order the versions were written. A proposal is a clock reading and cannot
+ * promise that: ids proposed inside one millisecond can exhaust the counter
+ * separating them, and a leader change onto a lagging clock proposes a lower
+ * value.
+ *
+ *
So a proposal is a floor. The applied id is the later of it and the id
+ * after the key's current version, which the write path already holds — no read
+ * of its own, no global state, and identical on every OM. Under a clock
+ * regression an affected key's ids climb by one until proposals overtake them
+ * again: the versions stay ordered and only the id's reading as a time
+ * degrades.
+ */
+public class VersionIdAllocator {
+
+ private final VersionIdGenerator generator;
+
+ public VersionIdAllocator(ConfigurationSource conf) {
+ this(VersionIdGenerator.fromConfiguration(conf));
+ }
+
+ public VersionIdAllocator(VersionIdGenerator generator) {
+ this.generator = generator;
+ }
+
+ public VersionIdGenerator getGenerator() {
+ return generator;
+ }
+
+ /** Proposes an id, on the OM that received the request. */
+ public long propose() {
+ final long versionId = generator.generateVersionId();
+ Preconditions.checkState(versionId > VersionIdGenerator.UNSET_VERSION_ID,
+ "%s proposed the reserved versionId %s",
+ generator.getClass().getName(), versionId);
+ return versionId;
+ }
+
+ /**
+ * @param proposedVersionId the id the request carried, which must be set:
+ * an unset field would otherwise be applied as the reserved unset value
+ * @param currentVersion the key's current version, or null when it has none.
+ * The write path holds it already, so no extra read is needed here.
+ * @return the versionId to freeze on the version being applied
+ */
+ public long allocate(long proposedVersionId, OmKeyInfo currentVersion) {
+ Preconditions.checkArgument(
+ proposedVersionId > VersionIdGenerator.UNSET_VERSION_ID,
+ "no versionId was proposed for this version; every request that can "
+ + "create one has to propose it in preExecute");
+
+ final long versionId =
+ generator.versionIdFor(proposedVersionId, currentVersion != null);
+
+ if (currentVersion == null) {
+ return versionId;
+ }
+
+ final Long currentVersionId = currentVersion.getVersionId();
+ if (currentVersionId == null) {
+ // Written before versioning was enabled, so it carries no id and the key
+ // has no other versions to order against.
+ return versionId;
+ }
+
+ return Math.max(versionId, currentVersionId + 1);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index 0d51c6e8ab25..2debef801034 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -89,6 +89,7 @@
* | Column Family | Mapping |
* |----------------------------------------------------------------------------------|
* | keyTable | /volume/bucket/key :- KeyInfo |
+ * | versionedKeyTable | /volume/bucket/key\0revVersionId :- KeyInfo |
* | deletedTable | /volume/bucket/key :- RepeatedKeyInfo |
* | openKeyTable | /volume/bucket/key/id :- KeyInfo |
* | multipartInfoTable | /volume/bucket/key/uploadId :- parts |
@@ -213,6 +214,21 @@ public final class OMDBDefinition extends DBDefinition.WithMap {
StringCodec.get(),
OmKeyInfo.getKeyTableCodec());
+ public static final String VERSIONED_KEY_TABLE = "versionedKeyTable";
+ /**
+ * versionedKeyTable: /volume/bucket/key\0revVersionId :- KeyInfo.
+ * Noncurrent object versions (including noncurrent delete markers) of
+ * versioning-enabled buckets; the current version stays in keyTable.
+ * revVersionId is the fixed-width hex of (Long.MAX_VALUE - versionId), so
+ * versions of a key are adjacent and ordered newest first. The separator is
+ * OM_VERSIONED_KEY_SEPARATOR (0x00), not '/', because OBJECT_STORE key names
+ * contain '/' verbatim.
+ */
+ public static final DBColumnFamilyDefinition VERSIONED_KEY_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(VERSIONED_KEY_TABLE,
+ StringCodec.get(),
+ OmKeyInfo.getKeyTableCodec());
+
public static final String DELETED_TABLE = "deletedTable";
/** deletedTable: /volume/bucket/key :- RepeatedKeyInfo (excludes fields only used in openKeyTable). */
public static final DBColumnFamilyDefinition DELETED_TABLE_DEF
@@ -370,6 +386,7 @@ public final class OMDBDefinition extends DBDefinition.WithMap {
TENANT_STATE_TABLE_DEF,
TRANSACTION_INFO_TABLE_DEF,
USER_TABLE_DEF,
+ VERSIONED_KEY_TABLE_DEF,
VOLUME_TABLE_DEF,
LIFECYCLE_CONFIGURATION_TABLE_DEF,
LIFECYCLE_SCAN_STATE_TABLE_DEF);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 94a328a77c17..ca5e1acf814c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -53,6 +53,7 @@
import org.apache.hadoop.ozone.om.request.file.OMRecoverLeaseRequest;
import org.apache.hadoop.ozone.om.request.key.OMDirectoriesPurgeRequestWithFSO;
import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
+import org.apache.hadoop.ozone.om.request.key.OMObjectVersionsReclaimRequest;
import org.apache.hadoop.ozone.om.request.key.OMOpenKeysDeleteRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequestWithFSO;
@@ -356,6 +357,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest,
return new OMLifecycleSetServiceStatusRequest(omRequest);
case SaveLifecycleScanState:
return new OMLifecycleSaveScanStateRequest(omRequest);
+ case ReclaimObjectVersions:
+ return new OMObjectVersionsReclaimRequest(omRequest);
case PutBucketTagging:
return new S3PutBucketTaggingRequest(omRequest);
case DeleteBucketTagging:
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 718f329aaaff..0451dc3a5b8a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -60,6 +60,7 @@
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketVersioningStatusProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -101,6 +102,23 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
// FSO and LEGACY buckets are not strictly bound to S3 naming semantics.
OmUtils.validateBucketName(bucketInfo.getBucketName(), strict);
+ // S3 has no way to create a bucket that is already in a versioning
+ // state; the state is set afterwards, through the transition checks in
+ // OMBucketSetPropertyRequest. versioningStatus is on BucketInfo because
+ // that message is also the bucket's on-disk record and the shape
+ // InfoBucket and ListBuckets return, not because a create needs it.
+ if (bucketInfo.hasVersioningStatus()) {
+ throw new OMException("Bucket versioning cannot be set while the bucket"
+ + " is being created; create the bucket first and then set its"
+ + " versioning status.", ResultCodes.INVALID_REQUEST);
+ }
+
+ if (bucketInfo.getVersioningStatus()
+ == BucketVersioningStatusProto.VERSIONING_ENABLED) {
+ OMBucketSetPropertyRequest.rejectIfLifecycleServiceIsDisabled(
+ ozoneManager);
+ }
+
// ACL check during preExecute
if (ozoneManager.getAclsEnabled()) {
try {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
index 7aa9dbe31d14..809a1091fc3c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
@@ -225,13 +225,22 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
}
- private boolean bucketContainsSnapshot(OMMetadataManager omMetadataManager,
+ /**
+ * Whether the bucket has any snapshot. A snapshot that was deleted but not
+ * yet purged is still in the table, so it counts as present: it keeps
+ * referencing the blocks it captured until the purge removes it.
+ *
+ * @param snapshotBucketKey the bucket key with a trailing '/', so that two
+ * buckets whose names share a prefix do not match
+ * each other.
+ */
+ static boolean bucketContainsSnapshot(OMMetadataManager omMetadataManager,
String snapshotBucketKey) throws IOException {
return bucketContainsSnapshotInCache(omMetadataManager, snapshotBucketKey)
|| bucketContainsSnapshotInTable(omMetadataManager, snapshotBucketKey);
}
- private boolean bucketContainsSnapshotInTable(
+ private static boolean bucketContainsSnapshotInTable(
OMMetadataManager omMetadataManager, String snapshotBucketKey)
throws IOException {
try (TableIterator> snapshotIterator
@@ -244,7 +253,7 @@ private boolean bucketContainsSnapshotInTable(
return false;
}
- private boolean bucketContainsSnapshotInCache(
+ private static boolean bucketContainsSnapshotInCache(
OMMetadataManager omMetadataManager, String snapshotBucketKey) {
Iterator, CacheValue>> cacheIter =
omMetadataManager.getSnapshotInfoTable().cacheIterator();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
index a88e5fb73334..ccbec21c3c5b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
@@ -32,12 +32,15 @@
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.common.BekInfoUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -49,9 +52,11 @@
import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.bucket.OMBucketSetPropertyResponse;
+import org.apache.hadoop.ozone.om.service.KeyLifecycleService;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketVersioningStatusProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
@@ -88,6 +93,11 @@ public OMRequest preExecute(OzoneManager ozoneManager)
BucketArgs bucketArgs =
getOmRequest().getSetBucketPropertyRequest().getBucketArgs();
+ if (bucketArgs.getVersioningStatus()
+ == BucketVersioningStatusProto.VERSIONING_ENABLED) {
+ rejectIfLifecycleServiceIsDisabled(ozoneManager);
+ }
+
if (bucketArgs.hasBekInfo()) {
KeyProviderCryptoExtension kmsProvider = ozoneManager.getKmsProvider();
BucketArgs.Builder bucketArgsBuilder =
@@ -174,10 +184,43 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
//Check Versioning to update
Boolean versioning = omBucketArgs.getIsVersionEnabled();
+ BucketVersioningStatus newVersioningStatus = omBucketArgs.getVersioningStatus();
if (versioning != null) {
+ // Apply the legacy flag on its own; setVersioningStatus below overrides
+ // it when a status is also being set.
bucketInfoBuilder.setIsVersionEnabled(versioning);
- LOG.debug("Updating bucket versioning for bucket: {} in volume: {}",
- bucketName, volumeName);
+ }
+ if (newVersioningStatus == null && versioning != null
+ && dbBucketInfo.hasVersioningStatus()) {
+ // Legacy flag from an older client against a bucket that already has an
+ // S3 versioning status: keep the two consistent. Disabling maps to
+ // SUSPENDED, since the S3 state machine forbids returning to
+ // UNVERSIONED once versioning has been enabled.
+ //
+ // On a bucket without a status the flag is left alone: it selects the
+ // legacy in-record block version list, and an old client must not be
+ // able to opt a bucket into S3 versioning semantics.
+ newVersioningStatus = versioning
+ ? BucketVersioningStatus.ENABLED : BucketVersioningStatus.SUSPENDED;
+ }
+ if (newVersioningStatus != null) {
+ if (dbBucketInfo.getBucketLayout() != BucketLayout.OBJECT_STORE) {
+ throw new OMException("S3 object versioning is only supported on "
+ + BucketLayout.OBJECT_STORE + " buckets, but bucket " + bucketName
+ + " has layout " + dbBucketInfo.getBucketLayout() + ".",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+ if (!dbBucketInfo.getVersioningStatus().canTransitionTo(newVersioningStatus)) {
+ throw new OMException("Bucket versioning cannot be changed from "
+ + dbBucketInfo.getVersioningStatus() + " to " + newVersioningStatus
+ + "; once enabled, versioning can only be suspended.",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ rejectVersioningWhileSnapshotsExist(ozoneManager, omMetadataManager,
+ dbBucketInfo, newVersioningStatus, volumeName, bucketName);
+ bucketInfoBuilder.setVersioningStatus(newVersioningStatus);
+ LOG.debug("Updating bucket versioning to {} for bucket: {} in volume: {}",
+ newVersioningStatus, bucketName, volumeName);
}
//Check quotaInBytes and quotaInNamespace to update
@@ -376,4 +419,103 @@ public static OMRequest disallowSetBucketPropertyWithECReplicationConfig(
}
return req;
}
+
+ /**
+ * Rejects a transition that leaves the bucket in a versioned state while it
+ * still holds snapshots. A snapshot shares blocks with the active object
+ * store and keeps them only because KeyDeletingService refuses to reclaim
+ * what a previous snapshot references; that decision is not version-aware
+ * yet, so a noncurrent version reclaimed here could take blocks a snapshot
+ * still points at.
+ *
+ * ENABLED -> SUSPENDED is allowed even then: a bucket that reached a
+ * mixed state before these checks existed has to be able to stop
+ * accumulating versions. A transition to the status the bucket already has
+ * is allowed for the same reason - it creates no new coexistence, and an S3
+ * client re-sending PutBucketVersioning must not fail against a bucket that
+ * is already in that state.
+ *
+ *
The check is not latched: once the last snapshot is purged from the
+ * bucket, versioning can be enabled or resumed normally.
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private void rejectVersioningWhileSnapshotsExist(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager, OmBucketInfo dbBucketInfo,
+ BucketVersioningStatus newVersioningStatus, String volumeName,
+ String bucketName) throws IOException {
+
+ if (ozoneManager.isSnapshotVersioningCoexistenceAllowed()) {
+ return;
+ }
+
+ BucketVersioningStatus current = dbBucketInfo.getVersioningStatus();
+ if (newVersioningStatus == BucketVersioningStatus.UNVERSIONED
+ || newVersioningStatus == current
+ || (current == BucketVersioningStatus.ENABLED
+ && newVersioningStatus == BucketVersioningStatus.SUSPENDED)) {
+ return;
+ }
+
+ // The trailing separator keeps two buckets whose names share a prefix from
+ // matching each other. A snapshot deleted but not yet purged is still in
+ // the table and still counts: it references its blocks until the purge.
+ String snapshotBucketKey = omMetadataManager.getBucketKey(volumeName,
+ bucketName) + OzoneConsts.OM_KEY_PREFIX;
+ if (OMBucketDeleteRequest.bucketContainsSnapshot(omMetadataManager,
+ snapshotBucketKey)) {
+ throw new OMException("Cannot set the object versioning status of bucket "
+ + bucketName + " to " + newVersioningStatus + " while it has"
+ + " snapshots. Snapshots and S3 object versioning cannot be used on"
+ + " the same bucket until snapshot-aware version reclamation is"
+ + " available; without it a noncurrent version reclaimed from the"
+ + " active object store would take blocks a snapshot still"
+ + " references. Delete and purge the bucket's snapshots first.",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+ }
+
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.SetBucketProperty
+ )
+ /**
+ * Versions are reclaimed by the lifecycle scan and by nothing else, so a
+ * bucket must not be able to enter a state where they accumulate with
+ * nothing able to reclaim them. Checked here rather than in
+ * validateAndUpdateCache because whether the service runs is a property of
+ * this OM's configuration, not of the replicated state: only the OM that
+ * received the request may decide it, and its decision is what travels.
+ */
+ static void rejectIfLifecycleServiceIsDisabled(OzoneManager ozoneManager)
+ throws OMException {
+ KeyLifecycleService lifecycleService =
+ ozoneManager.getKeyManager() == null ? null
+ : ozoneManager.getKeyManager().getKeyLifecycleService();
+ if (lifecycleService != null && !lifecycleService.isEnabled()) {
+ throw new OMException("S3 object versioning cannot be enabled while the "
+ + "lifecycle service is disabled: versions would accumulate with "
+ + "nothing able to reclaim them. Set "
+ + OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED + " to true.",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+ }
+
+ public static OMRequest disallowSetBucketPropertyWithVersioningStatus(
+ OMRequest req, ValidationContext ctx) throws OMException {
+ if (!ctx.versionManager()
+ .isAllowed(OMLayoutFeature.OBJECT_VERSIONING)) {
+ SetBucketPropertyRequest propReq =
+ req.getSetBucketPropertyRequest();
+ if (propReq.hasBucketArgs()
+ && propReq.getBucketArgs().hasVersioningStatus()) {
+ throw new OMException("Cluster does not have the object versioning"
+ + " feature finalized yet, but the request contains a bucket"
+ + " versioning status. Rejecting the request, please finalize the"
+ + " cluster upgrade and then try again.",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION);
+ }
+ }
+ return req;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/DeleteMarkerInsertion.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/DeleteMarkerInsertion.java
new file mode 100644
index 000000000000..ff5681067401
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/DeleteMarkerInsertion.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import java.util.Map;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+
+/**
+ * What inserting a delete marker changed, for the response to write out.
+ *
+ *
Produced by {@link OMKeyRequest#insertDeleteMarker}, which has already
+ * applied the records to the table cache. The namespace it adds is left to
+ * the caller to apply to the bucket, once every marker of the request is in:
+ * the bucket is the cached instance, so counting a marker of a batch that
+ * then fails would persist with the next write to it. A response only has to
+ * put the same records into its WriteBatch.
+ */
+public final class DeleteMarkerInsertion {
+
+ private final OmKeyInfo deleteMarker;
+ private final String objectKey;
+ private final String demotedVersionKey;
+ private final OmKeyInfo demotedVersion;
+ private final long addedNamespace;
+ private final long releasedBytes;
+ private final String replacedNullVersionKey;
+ private final Map keysToDelete;
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ DeleteMarkerInsertion(OmKeyInfo deleteMarker, String objectKey,
+ String demotedVersionKey, OmKeyInfo demotedVersion,
+ String replacedNullVersionKey,
+ Map keysToDelete, long addedNamespace,
+ long releasedBytes) {
+ this.deleteMarker = deleteMarker;
+ this.objectKey = objectKey;
+ this.demotedVersionKey = demotedVersionKey;
+ this.demotedVersion = demotedVersion;
+ this.replacedNullVersionKey = replacedNullVersionKey;
+ this.keysToDelete = keysToDelete;
+ this.addedNamespace = addedNamespace;
+ this.releasedBytes = releasedBytes;
+ }
+
+ /** Namespace the marker takes, not yet applied to the bucket. */
+ public long getAddedNamespace() {
+ return addedNamespace;
+ }
+
+ /** Space of the null version the marker replaced, not yet released. */
+ public long getReleasedBytes() {
+ return releasedBytes;
+ }
+
+ /**
+ * versionedKeyTable dbKey of the null version the marker replaced while
+ * versioning was suspended, or null when nothing was replaced.
+ */
+ public String getReplacedNullVersionKey() {
+ return replacedNullVersionKey;
+ }
+
+ /** Blocks of the replaced null version, queued for reclamation. */
+ public Map getKeysToDelete() {
+ return keysToDelete;
+ }
+
+ /** The marker that becomes the key's current version. */
+ public OmKeyInfo getDeleteMarker() {
+ return deleteMarker;
+ }
+
+ /** keyTable dbKey the marker takes. */
+ public String getObjectKey() {
+ return objectKey;
+ }
+
+ /**
+ * versionedKeyTable dbKey the superseded version moves to, or null when the
+ * key had no current version.
+ */
+ public String getDemotedVersionKey() {
+ return demotedVersionKey;
+ }
+
+ public OmKeyInfo getDemotedVersion() {
+ return demotedVersion;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 6c34443058b5..e72eaa90be91 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -34,6 +34,8 @@
import java.util.Map;
import java.util.Objects;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneManagerVersion;
@@ -52,6 +54,7 @@
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
import org.apache.hadoop.ozone.om.helpers.WithMetadata;
import org.apache.hadoop.ozone.om.request.util.OmKeyHSyncUtil;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
@@ -124,7 +127,12 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
KeyArgs.Builder newKeyArgs =
keyArgs.toBuilder().setModificationTime(Time.now())
- .setKeyName(keyPath);
+ .setKeyName(keyPath)
+ // Proposed here so every OM applies the same id. The call is local
+ // and cheap, so it is made for every commit and simply goes unused
+ // when the bucket turns out not to be versioned.
+ .setProposedVersionId(
+ ozoneManager.getVersionIdAllocator().propose());
KeyArgs resolvedKeyArgs =
resolveBucketAndCheckOpenKeyAcls(newKeyArgs.build(), ozoneManager,
@@ -303,29 +311,86 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap);
+ final boolean versioningEnabled = omBucketInfo.isS3VersioningEnabled();
+ final boolean keepsVersions = omBucketInfo.hasEverBeenVersioned();
+ // A write while versioning is suspended creates no version of its own: it
+ // takes the key's null version slot, replacing whatever held it. The
+ // record still carries a generated versionId, so that it sorts among the
+ // key's versions by the time it was written.
+ final boolean suspendedWrite = omBucketInfo.isS3VersioningSuspended();
// Set the UpdateID to current transactionLogIndex
- omKeyInfo = omKeyInfo.toBuilder()
+ OmKeyInfo.Builder committedKeyBuilder = omKeyInfo.toBuilder()
.setExpectedDataGeneration(null)
.addAllMetadata(KeyValueUtil.getFromProtobuf(
commitKeyArgs.getMetadataList()))
.setUpdateID(trxnLogIndex)
- .setDataSize(commitKeyArgs.getDataSize())
- .build();
+ .setDataSize(commitKeyArgs.getDataSize());
+ if (keepsVersions) {
+ // The version identity is frozen when the version is created: an hsync
+ // re-commit keeps updating the same version, so it keeps its versionId.
+ committedKeyBuilder.setVersionId(isSameHsyncKey
+ ? keyToDelete.getVersionId()
+ : ozoneManager.getVersionIdAllocator().allocate(
+ commitKeyArgs.getProposedVersionId(), keyToDelete));
+ committedKeyBuilder.setNullVersion(suspendedWrite);
+ }
+ omKeyInfo = committedKeyBuilder.build();
+ if (omKeyInfo.getVersionId() != null) {
+ // The id as applied, which the floor can raise above the proposal
+ // that the key args carry.
+ auditMap.put(OzoneConsts.VERSION_ID,
+ String.valueOf(omKeyInfo.getVersionId()));
+ auditMap.put(OzoneConsts.NULL_VERSION,
+ String.valueOf(omKeyInfo.isNullVersion()));
+ }
+
+ // The version a write supersedes is kept as a noncurrent version, except
+ // for the null version, which a suspended write replaces outright.
+ final boolean supersededVersionRetained = keyToDelete != null
+ && keepsVersions
+ && (versioningEnabled || !keyToDelete.isNullVersionRecord());
// Update the block length for each block, return the allocated but
// uncommitted blocks
List uncommitted =
omKeyInfo.updateLocationInfoList(locationInfoList, false);
+ // A suspended write replaces the key's null version wherever it is. It is
+ // the current version when the previous write was also suspended, and a
+ // noncurrent version when versioning was enabled in between - the version
+ // that demoted it is still current, and is retained below.
+ //
+ // Resolved before the quota checks so that they see the whole
+ // transaction. The replaced record goes away in the same batch, so a
+ // write that leaves the bucket no larger must not be rejected for being
+ // over quota. The version retained below always carries a real versionId,
+ // since a null version record would not have been retained, so it cannot
+ // be what this finds.
+ final Pair replacedNullVersion =
+ suspendedWrite && !isSameHsyncKey && supersededVersionRetained
+ ? getNoncurrentNullVersion(ozoneManager,
+ omMetadataManager, volumeName, bucketName, keyName)
+ : null;
+ // The write adds a record, unless it also removes one: replacing the
+ // key's null version leaves the number of records unchanged.
+ final long addedNamespace = replacedNullVersion == null ? 1L : 0L;
+
Map oldKeyVersionsToDeleteMap = null;
long correctedSpace = omKeyInfo.getReplicatedSize();
+ if (replacedNullVersion != null) {
+ correctedSpace -= sumBlockLengths(replacedNullVersion.getValue());
+ }
// if keyToDelete isn't null, usedNamespace needn't check and
// increase.
if (keyToDelete != null && (isSameHsyncKey)) {
correctedSpace -= keyToDelete.getReplicatedSize();
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
- } else if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
+ } else if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()
+ && !supersededVersionRetained) {
+ // A retained version keeps its blocks: it lives on in the
+ // versionedKeyTable. What reaches this branch under S3 versioning is
+ // the null version being replaced by a suspended write.
RepeatedOmKeyInfo oldVerKeyInfo = getOldVersionsToCleanUp(
keyToDelete, omBucketInfo.getObjectID(), trxnLogIndex);
// using pseudoObjId as objectId can be same in case of overwrite key
@@ -368,11 +433,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
omBucketInfo.decrUsedNamespace(filteredUsedBlockCnt.getRight(), false);
omBucketInfo.decrUsedBytes(totalSize, true);
} else {
- checkBucketQuotaInNamespace(omBucketInfo, 1L);
+ checkBucketQuotaInNamespace(omBucketInfo, addedNamespace);
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
correctedSpace);
}
- omBucketInfo.incrUsedNamespace(1L);
+ omBucketInfo.incrUsedNamespace(addedNamespace);
// let the uncommitted blocks pretend as key's old version blocks
// which will be deleted as RepeatedOmKeyInfo
final OmKeyInfo pseudoKeyInfo = isHSync ? null
@@ -401,6 +466,47 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
dbOpenKey, newOpenKeyInfo, trxnLogIndex);
}
+ // With S3-compatible versioning the overwritten current version is kept
+ // as a noncurrent version instead of being reclaimed. A record written
+ // before versioning was enabled carries no versionId and becomes the
+ // key's null version.
+ String dbVersionedKey = null;
+ OmKeyInfo versionedKeyInfo = null;
+ if (supersededVersionRetained && !isSameHsyncKey) {
+ // This commit is what supersedes it, so this is when it stopped being
+ // current.
+ OmKeyInfo.Builder versionedBuilder = keyToDelete.toBuilder()
+ .setNoncurrentTime(commitKeyArgs.getModificationTime());
+ if (keyToDelete.getVersionId() == null) {
+ versionedBuilder
+ .setVersionId(VersionIdGenerator.UNSET_VERSION_ID)
+ .setNullVersion(true);
+ }
+ versionedKeyInfo = versionedBuilder.build();
+ dbVersionedKey = omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionedKeyInfo.getVersionId());
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ dbVersionedKey, versionedKeyInfo, trxnLogIndex);
+ auditMap.put(OzoneConsts.SUPERSEDED_VERSION_ID,
+ String.valueOf(versionedKeyInfo.getVersionId()));
+ }
+
+ // A suspended write replaces the key's null version wherever it is. It is
+ // the current version when the previous write was also suspended, and a
+ // noncurrent version when versioning was enabled in between - the version
+ // that demoted it is still current, and is retained above.
+ String replacedNullVersionKey = null;
+ if (replacedNullVersion != null) {
+ replacedNullVersionKey = replacedNullVersion.getKey();
+ oldKeyVersionsToDeleteMap = addKeyInfoToDeleteMap(ozoneManager,
+ trxnLogIndex, dbOzoneKey, omBucketInfo.getObjectID(),
+ replacedNullVersion.getValue().withCommittedKeyDeletedFlag(true),
+ oldKeyVersionsToDeleteMap);
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(replacedNullVersionKey),
+ CacheValue.get(trxnLogIndex));
+ }
+
omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
dbOzoneKey, omKeyInfo, trxnLogIndex);
@@ -408,7 +514,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
omClientResponse = new OMKeyCommitResponse(omResponse.build(),
omKeyInfo, dbOzoneKey, dbOpenKey, omBucketInfo.copyObject(),
- oldKeyVersionsToDeleteMap, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete);
+ oldKeyVersionsToDeleteMap, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete)
+ .withVersionedKey(dbVersionedKey, versionedKeyInfo)
+ .withReplacedNullVersion(replacedNullVersionKey);
result = Result.SUCCESS;
} catch (IOException | InvalidPathException ex) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index 26287ca66d26..a13e405f57ba 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -19,6 +19,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
@@ -41,13 +42,16 @@
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteMarkerResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyVersionDeleteResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyResponse;
@@ -80,13 +84,21 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
Objects.requireNonNull(deleteKeyRequest, "deleteKeyRequest == null");
OzoneManagerProtocolProtos.KeyArgs keyArgs = deleteKeyRequest.getKeyArgs();
+ OmKeyArgs.validateAddressedVersion(keyArgs);
+
String keyPath = keyArgs.getKeyName();
OmUtils.verifyKeyNameWithSnapshotReservedWordForDeletion(keyPath);
keyPath = normalizeKeyPath(ozoneManager.getEnableFileSystemPaths(), keyPath, getBucketLayout());
OzoneManagerProtocolProtos.KeyArgs.Builder newKeyArgs =
- keyArgs.toBuilder().setModificationTime(Time.now()).setKeyName(keyPath);
+ keyArgs.toBuilder().setModificationTime(Time.now()).setKeyName(keyPath)
+ // A delete without a versionId on a versioned bucket creates a
+ // delete marker, and a marker is a version, so it needs an id
+ // proposed here on the OM that received the request. Unused on an
+ // unversioned bucket, and by a delete that addresses a version.
+ .setProposedVersionId(
+ ozoneManager.getVersionIdAllocator().propose());
KeyArgs resolvedArgs = resolveBucketAndCheckAcls(ozoneManager, newKeyArgs);
return getOmRequest().toBuilder()
@@ -110,6 +122,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
OzoneManagerProtocolProtos.KeyArgs keyArgs = deleteKeyRequest.getKeyArgs();
Map auditMap = buildLightKeyArgsAuditMap(keyArgs);
+ // The version the request addresses, recorded up front so that a delete
+ // that fails still shows what it was for.
+ if (keyArgs.hasVersionId()) {
+ auditMap.put(OzoneConsts.REQUESTED_VERSION_ID,
+ String.valueOf(keyArgs.getVersionId()));
+ } else if (keyArgs.getNullVersion()) {
+ auditMap.put(OzoneConsts.REQUESTED_VERSION_ID, "null");
+ }
String volumeName = keyArgs.getVolumeName();
String bucketName = keyArgs.getBucketName();
@@ -128,6 +148,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
boolean acquiredLock = false;
OMClientResponse omClientResponse = null;
Result result = null;
+ // whether the request removed a key that was visible to plain reads; a
+ // delete marker supersedes the current version without removing anything
+ boolean visibleKeyRemoved = false;
+ // whether the request took the delete marker path, and so may have left
+ // versionedKeyTable entries in the table cache for this transaction
+ boolean insertingDeleteMarker = false;
+ // whether the request took the permanent version delete path, and so may
+ // have left versionedKeyTable entries in the table cache
+ boolean deletingVersion = false;
long startNanos = Time.monotonicNowNanos();
try {
String objectKey =
@@ -142,57 +171,117 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
OmKeyInfo omKeyInfo =
omMetadataManager.getKeyTable(getBucketLayout()).get(objectKey);
- if (omKeyInfo == null) {
- throw new OMException("Key not found", KEY_NOT_FOUND);
- }
-
- validateIfMatchETag(keyArgs, omKeyInfo);
-
- // Set the UpdateID to current transactionLogIndex
- omKeyInfo = omKeyInfo.toBuilder()
- .setUpdateID(trxnLogIndex)
- .build();
-
- // Update table cache. Put a tombstone entry
- omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
- new CacheKey<>(
- omMetadataManager.getOzoneKey(volumeName, bucketName, keyName)),
- CacheValue.get(trxnLogIndex));
OmBucketInfo omBucketInfo =
getBucketInfo(omMetadataManager, volumeName, bucketName);
- long quotaReleased = sumBlockLengths(omKeyInfo);
- // Empty entries won't be added to deleted table so this key shouldn't get added to snapshotUsed space.
- boolean isKeyNonEmpty = !OmKeyInfo.isKeyEmpty(omKeyInfo);
- omBucketInfo.decrUsedBytes(quotaReleased, isKeyNonEmpty);
- omBucketInfo.decrUsedNamespace(1L, isKeyNonEmpty);
- OmKeyInfo deletedOpenKeyInfo = null;
-
- // If omKeyInfo has hsync metadata, delete its corresponding open key as well
- String dbOpenKey = null;
- String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID);
- if (hsyncClientId != null) {
- Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout());
- dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId);
- OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
- if (openKeyInfo != null) {
- openKeyInfo = openKeyInfo.withMetadataMutations(
- metadata -> metadata.put(DELETED_HSYNC_KEY, "true"));
- openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
- deletedOpenKeyInfo = openKeyInfo;
- } else {
- LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey);
+ final boolean addressesVersion =
+ keyArgs.hasVersionId() || keyArgs.getNullVersion();
+ if (addressesVersion) {
+ // Only OBJECT_STORE buckets can hold versions, so addressing one
+ // anywhere else is not supported, as on the read path.
+ if (omBucketInfo.getBucketLayout() != BucketLayout.OBJECT_STORE) {
+ throw new OMException("Object versioning is only supported on "
+ + BucketLayout.OBJECT_STORE + " buckets, but bucket "
+ + bucketName + " has layout " + omBucketInfo.getBucketLayout()
+ + ".", NOT_SUPPORTED_OPERATION);
+ }
+ // A bucket that has never been versioned holds no version an id could
+ // name. It does hold the key's null version, though: S3 calls the
+ // object itself the null version wherever there is no version history,
+ // so a delete that addresses it is the plain delete below rather than
+ // a version delete at all.
+ //
+ // Reported as not-found rather than unsupported: S3 answers a delete
+ // naming a version that does not exist with a plain success, which the
+ // gateway reaches by swallowing KEY_NOT_FOUND. NOT_SUPPORTED_OPERATION
+ // would surface an error AWS never returns here.
+ if (keyArgs.hasVersionId() && !omBucketInfo.hasEverBeenVersioned()) {
+ throw new OMException("Bucket " + bucketName
+ + " does not have S3 versioning enabled, so it holds no version "
+ + "the request could name", KEY_NOT_FOUND);
}
}
- omClientResponse = new OMKeyDeleteResponse(
- omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder())
- .build(), omKeyInfo,
- omBucketInfo.copyObject(), deletedOpenKeyInfo);
- if (omKeyInfo.isFile()) {
- auditMap.put(OzoneConsts.DATA_SIZE, String.valueOf(omKeyInfo.getDataSize()));
- auditMap.put(OzoneConsts.REPLICATION_CONFIG, omKeyInfo.getReplicationConfig().toString());
+ if (addressesVersion && omBucketInfo.hasEverBeenVersioned()) {
+ // DELETE ?versionId= permanently removes one version. It is the only
+ // delete that destroys data on a versioned bucket.
+ deletingVersion = true;
+ omClientResponse = deleteVersion(ozoneManager, omMetadataManager,
+ omBucketInfo, omKeyInfo, volumeName, bucketName, keyName,
+ keyArgs.getVersionId(), keyArgs.getNullVersion(), trxnLogIndex,
+ omResponse, auditMap);
+ // Noncurrent versions are invisible to plain reads, so removing one
+ // does not change the visible key count. Deleting the current one
+ // with nothing left to promote takes the key away entirely, and the
+ // record that was there is what stops being visible.
+ visibleKeyRemoved = omKeyInfo != null && !omKeyInfo.isDeleteMarker()
+ && omMetadataManager.getKeyTable(getBucketLayout())
+ .get(objectKey) == null;
+ } else if (omBucketInfo.hasEverBeenVersioned()) {
+ // A delete without a versionId removes no data: a delete marker
+ // becomes the current version and the version it supersedes moves to
+ // the versionedKeyTable. Like S3, the marker is inserted even when the
+ // key does not exist. While versioning is suspended the marker takes
+ // the key's null version slot instead of creating a version.
+ insertingDeleteMarker = true;
+ omClientResponse = insertDeleteMarker(ozoneManager, omMetadataManager,
+ omBucketInfo, omKeyInfo, objectKey, keyArgs, trxnLogIndex,
+ omResponse, auditMap);
+ // The key stays in the keyTable as a marker, so nothing is released;
+ // only superseding a visible object is one fewer visible key.
+ visibleKeyRemoved = omKeyInfo != null && !omKeyInfo.isDeleteMarker();
+ } else {
+ if (omKeyInfo == null) {
+ throw new OMException("Key not found", KEY_NOT_FOUND);
+ }
+
+ validateIfMatchETag(keyArgs, omKeyInfo);
+
+ // Set the UpdateID to current transactionLogIndex
+ omKeyInfo = omKeyInfo.toBuilder()
+ .setUpdateID(trxnLogIndex)
+ .build();
+
+ // Update table cache. Put a tombstone entry
+ omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
+ new CacheKey<>(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName)),
+ CacheValue.get(trxnLogIndex));
+
+ long quotaReleased = sumBlockLengths(omKeyInfo);
+ // Empty entries won't be added to deleted table so this key shouldn't get added to snapshotUsed space.
+ boolean isKeyNonEmpty = !OmKeyInfo.isKeyEmpty(omKeyInfo);
+ omBucketInfo.decrUsedBytes(quotaReleased, isKeyNonEmpty);
+ omBucketInfo.decrUsedNamespace(1L, isKeyNonEmpty);
+ OmKeyInfo deletedOpenKeyInfo = null;
+
+ // If omKeyInfo has hsync metadata, delete its corresponding open key as well
+ String dbOpenKey = null;
+ String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID);
+ if (hsyncClientId != null) {
+ Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout());
+ dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId);
+ OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
+ if (openKeyInfo != null) {
+ openKeyInfo = openKeyInfo.withMetadataMutations(
+ metadata -> metadata.put(DELETED_HSYNC_KEY, "true"));
+ openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
+ deletedOpenKeyInfo = openKeyInfo;
+ } else {
+ LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey);
+ }
+ }
+
+ omClientResponse = new OMKeyDeleteResponse(
+ omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder())
+ .build(), omKeyInfo,
+ omBucketInfo.copyObject(), deletedOpenKeyInfo);
+ if (omKeyInfo.isFile()) {
+ auditMap.put(OzoneConsts.DATA_SIZE, String.valueOf(omKeyInfo.getDataSize()));
+ auditMap.put(OzoneConsts.REPLICATION_CONFIG, omKeyInfo.getReplicationConfig().toString());
+ }
+ visibleKeyRemoved = true;
}
result = Result.SUCCESS;
@@ -201,9 +290,22 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
} catch (IOException | InvalidPathException ex) {
result = Result.FAILURE;
exception = ex;
- omClientResponse =
- new OMKeyDeleteResponse(createErrorOMResponse(omResponse, exception),
- getBucketLayout());
+ // The failure response has to declare the same tables as the successful
+ // one: the double buffer cleans up the table cache from the response's
+ // CleanupTableInfo, so a delete marker request that failed after
+ // touching the versionedKeyTable cache would otherwise leave an entry
+ // behind that is in no DB and is never cleaned up.
+ OMResponse errorResponse = createErrorOMResponse(omResponse, exception);
+ if (insertingDeleteMarker) {
+ omClientResponse =
+ new OMKeyDeleteMarkerResponse(errorResponse, getBucketLayout());
+ } else if (deletingVersion) {
+ omClientResponse =
+ new OMKeyVersionDeleteResponse(errorResponse, getBucketLayout());
+ } else {
+ omClientResponse =
+ new OMKeyDeleteResponse(errorResponse, getBucketLayout());
+ }
long endNanosDeleteKeyFailureLatencyNs = Time.monotonicNowNanos();
perfMetrics.setDeleteKeyFailureLatencyNs(endNanosDeleteKeyFailureLatencyNs - startNanos);
} finally {
@@ -222,7 +324,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
switch (result) {
case SUCCESS:
- omMetrics.decNumKeys();
+ if (visibleKeyRemoved) {
+ omMetrics.decNumKeys();
+ }
LOG.debug("Key deleted. Volume:{}, Bucket:{}, Key:{}", volumeName,
bucketName, keyName);
break;
@@ -239,6 +343,48 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
return omClientResponse;
}
+ /**
+ * Supersedes the key's current version with a delete marker: a record with
+ * no data blocks that makes plain reads of the key return KEY_NOT_FOUND
+ * while every existing version stays readable by versionId. The superseded
+ * current version becomes a noncurrent version; a record written before
+ * versioning was enabled carries no versionId and becomes the key's null
+ * version. When the key does not exist the marker is still inserted, as S3
+ * does.
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private OMClientResponse insertDeleteMarker(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager, OmBucketInfo omBucketInfo,
+ OmKeyInfo currentVersion, String objectKey,
+ OzoneManagerProtocolProtos.KeyArgs keyArgs, long trxnLogIndex,
+ OMResponse.Builder omResponse, Map auditMap)
+ throws IOException {
+
+ DeleteMarkerInsertion inserted = insertDeleteMarker(ozoneManager,
+ omMetadataManager, omBucketInfo, currentVersion, objectKey,
+ keyArgs.getKeyName(), keyArgs.getProposedVersionId(),
+ keyArgs.getModificationTime(), trxnLogIndex, 0L);
+ omBucketInfo.incrUsedNamespace(inserted.getAddedNamespace());
+ if (inserted.getReleasedBytes() > 0) {
+ omBucketInfo.decrUsedBytes(inserted.getReleasedBytes(), true);
+ }
+ auditMap.put(OzoneConsts.VERSION_ID,
+ String.valueOf(inserted.getDeleteMarker().getVersionId()));
+ auditMap.put(OzoneConsts.NULL_VERSION,
+ String.valueOf(inserted.getDeleteMarker().isNullVersion()));
+ if (inserted.getDemotedVersion() != null) {
+ auditMap.put(OzoneConsts.SUPERSEDED_VERSION_ID,
+ String.valueOf(inserted.getDemotedVersion().getVersionId()));
+ }
+
+ return new OMKeyDeleteMarkerResponse(
+ omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(),
+ inserted.getDeleteMarker(), inserted.getObjectKey(),
+ inserted.getDemotedVersionKey(), inserted.getDemotedVersion(),
+ omBucketInfo.copyObject(), inserted.getReplacedNullVersionKey(),
+ inserted.getKeysToDelete());
+ }
+
/**
* Validates key delete requests.
* We do not want to allow older clients to delete keys in buckets which use
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index ff65b97d0afe..68f433b3c39b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -26,6 +26,7 @@
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
import static org.apache.hadoop.ozone.om.helpers.OzoneAclUtil.getDefaultAclList;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
@@ -56,10 +57,12 @@
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -73,6 +76,7 @@
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.NoncurrentVersions;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -93,12 +97,16 @@
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
import org.apache.hadoop.ozone.om.lock.OzoneLockStrategy;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.OMClientRequestUtils;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.om.response.key.OMKeyVersionDeleteResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -944,6 +952,30 @@ public static long sumBlockLengths(OmKeyInfo omKeyInfo) {
return bytesUsed;
}
+ /**
+ * Returns the newest noncurrent version of the given key as a
+ * (dbKey, keyInfo) pair, or null if the key has no noncurrent version.
+ */
+ protected Pair getNewestNoncurrentVersion(
+ OzoneManager ozoneManager, OMMetadataManager omMetadataManager,
+ String volumeName, String bucketName, String keyName)
+ throws IOException {
+ return NoncurrentVersions.newestMatching(ozoneManager, omMetadataManager,
+ volumeName, bucketName, keyName, keyInfo -> true);
+ }
+
+ /**
+ * Returns the key's null version as a (dbKey, keyInfo) pair, or null if the
+ * key has no noncurrent null version. A key has at most one.
+ */
+ protected Pair getNoncurrentNullVersion(
+ OzoneManager ozoneManager, OMMetadataManager omMetadataManager,
+ String volumeName, String bucketName, String keyName)
+ throws IOException {
+ return NoncurrentVersions.nullVersion(ozoneManager, omMetadataManager,
+ volumeName, bucketName, keyName);
+ }
+
/**
* Return bucket info for the specified bucket.
*
@@ -1013,11 +1045,15 @@ protected OmKeyInfo prepareFileInfo(
if (dbKeyInfo != null) {
// The key already exist, the new blocks will replace old ones
// as new versions unless the bucket does not have versioning
- // turned on.
- dbKeyInfo.addNewVersion(locations, false,
- omBucketInfo.getIsVersionEnabled());
+ // turned on. With S3-compatible versioning the previous current version
+ // is kept as its own record in the versionedKeyTable at commit time, so
+ // the in-record block version list is not used to accumulate object
+ // versions and always holds a single version.
+ boolean keepInRecordVersions = omBucketInfo.getIsVersionEnabled()
+ && !omBucketInfo.hasEverBeenVersioned();
+ dbKeyInfo.addNewVersion(locations, false, keepInRecordVersions);
long newSize = size;
- if (omBucketInfo.getIsVersionEnabled()) {
+ if (keepInRecordVersions) {
newSize += dbKeyInfo.getDataSize();
}
// The modification time is set in preExecute. Use the same
@@ -1443,4 +1479,266 @@ protected KeyArgs validateAndRewriteIfMatchAsExpectedGeneration(
.clearExpectedETag()
.build();
}
+
+ /**
+ * Supersedes the key's current version with a delete marker: a record with
+ * no data blocks that makes plain reads of the key return KEY_NOT_FOUND
+ * while every existing version stays readable by versionId. The superseded
+ * current version becomes a noncurrent version; a record written before
+ * versioning was enabled carries no versionId and becomes the key's null
+ * version. When the key does not exist the marker is still inserted, as S3
+ * does.
+ *
+ *
Applies everything to the table cache and to the bucket's quota, and
+ * returns what a response has to put into its WriteBatch. Shared so that a
+ * single delete and a batch delete insert the same marker.
+ *
+ * @param currentVersion the key's current version, or null if it has none
+ * @param proposedVersionId the id the request proposed for the marker
+ * @param modificationTime the time to stamp the marker with
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected DeleteMarkerInsertion insertDeleteMarker(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager, OmBucketInfo omBucketInfo,
+ OmKeyInfo currentVersion, String objectKey, String keyName,
+ long proposedVersionId, long modificationTime, long trxnLogIndex,
+ long pendingNamespace)
+ throws IOException {
+
+ String volumeName = omBucketInfo.getVolumeName();
+ String bucketName = omBucketInfo.getBucketName();
+ // While versioning is suspended the marker is the key's null version, so
+ // it replaces whatever held that slot instead of superseding it.
+ final boolean suspended = omBucketInfo.isS3VersioningSuspended();
+ final boolean replacesCurrent = suspended && currentVersion != null
+ && currentVersion.isNullVersionRecord();
+
+ long markerVersionId = ozoneManager.getVersionIdAllocator().allocate(
+ proposedVersionId, currentVersion);
+
+ // Everything that can fail runs before the first cache entry is added: a
+ // request that throws here is answered with an OMKeyDeleteResponse, whose
+ // cleanup does not cover the versionedKeyTable, so a cache entry left
+ // behind would never be removed and would outlive the failed request.
+ // The null version the marker replaces is removed: the current one when
+ // the last write was also suspended, and a noncurrent one when versioning
+ // was enabled in between. Resolved before the quota check so that the
+ // check sees what the marker removes as well as what it adds.
+ final Pair noncurrentNullVersion =
+ suspended && !replacesCurrent
+ ? getNoncurrentNullVersion(ozoneManager,
+ omMetadataManager, volumeName, bucketName, keyName)
+ : null;
+ final OmKeyInfo replacedNullVersion = replacesCurrent ? currentVersion
+ : (noncurrentNullVersion == null
+ ? null : noncurrentNullVersion.getValue());
+ // The marker is a record of its own; it holds no blocks, so it consumes
+ // namespace but no space - unless it takes the null version slot, which
+ // replaces a record rather than adding one.
+ // The check also counts what earlier markers of the same request have
+ // claimed, which the bucket does not show yet.
+ final long addedNamespace = replacedNullVersion == null ? 1L : 0L;
+ checkBucketQuotaInNamespace(omBucketInfo,
+ pendingNamespace + addedNamespace);
+
+ // The marker is built from a stated set of fields rather than copied from
+ // the version it supersedes. It keeps the key's identity - objectID,
+ // owner, replication and ACLs - and holds nothing else: no blocks, no
+ // metadata or tags, no encryption info or checksum, which describe content
+ // a marker does not have. Listing the fields rather than copying and
+ // clearing also keeps a field added to OmKeyInfo off markers by default.
+ OmKeyInfo.Builder markerBuilder = new OmKeyInfo.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setOmKeyLocationInfos(Collections.singletonList(
+ new OmKeyLocationInfoGroup(0, new ArrayList<>())))
+ .setDataSize(0L)
+ .setCreationTime(modificationTime)
+ .setModificationTime(modificationTime)
+ .setUpdateID(trxnLogIndex)
+ .setFile(true)
+ .setVersionId(markerVersionId)
+ .setDeleteMarker(true)
+ .setNullVersion(suspended);
+ if (currentVersion != null) {
+ markerBuilder
+ .setObjectID(currentVersion.getObjectID())
+ .setOwnerName(currentVersion.getOwnerName())
+ .setReplicationConfig(currentVersion.getReplicationConfig())
+ .setAcls(currentVersion.getAcls());
+ } else {
+ // S3 inserts a delete marker for a key that does not exist, so the
+ // marker has no identity to inherit and gets one of its own.
+ markerBuilder
+ .setObjectID(ozoneManager.getObjectIdFromTxId(trxnLogIndex))
+ .setOwnerName(omBucketInfo.getOwner())
+ .setReplicationConfig(RatisReplicationConfig.getInstance(
+ ReplicationFactor.ONE));
+ }
+ OmKeyInfo deleteMarker = markerBuilder.build();
+
+ String movedVersionedKeyName = null;
+ OmKeyInfo movedVersionedKeyInfo = null;
+ if (currentVersion != null && !replacesCurrent) {
+ // The marker is what supersedes it, so this is when it stopped being
+ // current.
+ OmKeyInfo.Builder movedBuilder = currentVersion.toBuilder()
+ .setNoncurrentTime(modificationTime);
+ if (currentVersion.getVersionId() == null) {
+ movedBuilder
+ .setVersionId(VersionIdGenerator.UNSET_VERSION_ID)
+ .setNullVersion(true);
+ }
+ movedVersionedKeyInfo = movedBuilder.build();
+ movedVersionedKeyName = omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName,
+ movedVersionedKeyInfo.getVersionId());
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ movedVersionedKeyName, movedVersionedKeyInfo, trxnLogIndex);
+ }
+
+ String replacedNullVersionKey = null;
+ if (noncurrentNullVersion != null) {
+ replacedNullVersionKey = noncurrentNullVersion.getKey();
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(replacedNullVersionKey),
+ CacheValue.get(trxnLogIndex));
+ }
+ Map keysToDelete = null;
+ if (replacedNullVersion != null) {
+ keysToDelete = addKeyInfoToDeleteMap(ozoneManager, trxnLogIndex,
+ objectKey, omBucketInfo.getObjectID(),
+ replacedNullVersion.withCommittedKeyDeletedFlag(true), null);
+ }
+ // Released to the bucket by the caller, like the namespace, so that a
+ // batch failing on a later key leaves the cached bucket as it found it.
+ final long releasedBytes = replacedNullVersion == null ? 0L
+ : sumBlockLengths(replacedNullVersion);
+
+ omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
+ objectKey, deleteMarker, trxnLogIndex);
+
+ // The namespace is left to the caller: the bucket is the cached instance,
+ // and a batch that fails on a later key must not have counted this one.
+ return new DeleteMarkerInsertion(deleteMarker, objectKey,
+ movedVersionedKeyName, movedVersionedKeyInfo, replacedNullVersionKey,
+ keysToDelete, addedNamespace, releasedBytes);
+ }
+
+ /**
+ * Permanently removes the addressed version: the only delete that destroys
+ * data on a versioned bucket. Removing the current version hands its keyTable
+ * place over to the next-newest version of the key, or removes the key
+ * outright when none survives. Shared so that a single delete and a batch
+ * delete remove a version the same way.
+ *
+ * @param currentVersion the key's current version, or null if it has none
+ * @param auditMap where to record the versions involved, or null
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected OMKeyVersionDeleteResponse deleteVersion(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager,
+ OmBucketInfo omBucketInfo, OmKeyInfo currentVersion,
+ String volumeName, String bucketName, String keyName, long versionId,
+ boolean nullVersion, long trxnLogIndex,
+ OMResponse.Builder omResponse, Map auditMap)
+ throws IOException {
+
+ boolean deletingCurrent = currentVersion != null && (nullVersion
+ ? currentVersion.isNullVersionRecord()
+ : Long.valueOf(versionId).equals(currentVersion.getVersionId()));
+
+ String objectKey =
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
+
+ // The dbKey the version is deleted from, in the keyTable when it is the
+ // current version and in the versionedKeyTable otherwise. Everything that
+ // can fail runs before the first cache entry is added, so that a failed
+ // request leaves no versionedKeyTable cache entry behind.
+ String deletedVersionKey;
+ OmKeyInfo version;
+ if (deletingCurrent) {
+ deletedVersionKey = objectKey;
+ version = currentVersion;
+ } else if (nullVersion) {
+ Pair nullSlot = getNoncurrentNullVersion(
+ ozoneManager, omMetadataManager, volumeName, bucketName, keyName);
+ deletedVersionKey = nullSlot == null ? null : nullSlot.getKey();
+ version = nullSlot == null ? null : nullSlot.getValue();
+ } else {
+ deletedVersionKey = omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId);
+ version =
+ omMetadataManager.getVersionedKeyTable().get(deletedVersionKey);
+ }
+ if (version == null) {
+ throw new OMException("Version not found for key " + keyName,
+ KEY_NOT_FOUND);
+ }
+
+ // Removing the current version leaves the key without one, so the newest
+ // noncurrent version is promoted to keep the invariant that keyTable holds
+ // the current version of every key that still has one. The record moves
+ // unchanged: promotion is positional, the version keeps its identity.
+ String promotedKey = null;
+ OmKeyInfo promoted = null;
+ if (deletingCurrent) {
+ Pair newest = getNewestNoncurrentVersion(
+ ozoneManager, omMetadataManager, volumeName, bucketName, keyName);
+ if (newest != null) {
+ promotedKey = newest.getKey();
+ // Current again, so it has no moment of becoming noncurrent.
+ promoted = newest.getValue().toBuilder()
+ .setNoncurrentTime(0L)
+ .build();
+ }
+ }
+
+ version = version.toBuilder().setUpdateID(trxnLogIndex).build();
+
+ if (deletingCurrent) {
+ if (promoted != null) {
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .addCacheEntry(objectKey, promoted, trxnLogIndex);
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(promotedKey), CacheValue.get(trxnLogIndex));
+ } else {
+ // no version survives, so the key disappears entirely
+ omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
+ new CacheKey<>(objectKey), CacheValue.get(trxnLogIndex));
+ }
+ } else {
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(deletedVersionKey), CacheValue.get(trxnLogIndex));
+ }
+
+ if (auditMap != null) {
+ auditMap.put(OzoneConsts.DELETED_VERSION_ID,
+ String.valueOf(version.getVersionId()));
+ if (promoted != null) {
+ auditMap.put(OzoneConsts.PROMOTED_VERSION_ID,
+ String.valueOf(promoted.getVersionId()));
+ }
+ }
+
+ // A delete marker holds no blocks, so it releases namespace but no space.
+ long quotaReleased = sumBlockLengths(version);
+ boolean isVersionNonEmpty = !OmKeyInfo.isKeyEmpty(version);
+ omBucketInfo.decrUsedBytes(quotaReleased, isVersionNonEmpty);
+ omBucketInfo.decrUsedNamespace(1L, isVersionNonEmpty);
+
+ // Named by this transaction rather than the version's objectID: every
+ // version of a key carries the objectID of the record it overwrote, so two
+ // version deletes named by objectID would land on one deletedTable entry,
+ // and the later put would drop the earlier version's blocks for good.
+ String deletedTableKey = omMetadataManager.getOzoneDeletePathKey(
+ ozoneManager.getObjectIdFromTxId(trxnLogIndex), objectKey);
+
+ return new OMKeyVersionDeleteResponse(
+ omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(),
+ version, deletedVersionKey, deletingCurrent,
+ promotedKey, promoted, omBucketInfo.copyObject(), deletedTableKey);
+ }
+
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
index 3fc15da75ee9..b5140b934b5a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
@@ -38,9 +38,11 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -65,11 +67,15 @@
import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyVersionDeleteResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteMarkerResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyVersion;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource;
@@ -98,6 +104,14 @@ public OMKeysDeleteRequest(OMRequest omRequest, BucketLayout bucketLayout) {
public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
DeleteKeysRequest deleteKeysRequest = super.preExecute(ozoneManager).getDeleteKeysRequest();
Objects.requireNonNull(deleteKeysRequest, "deleteKeysRequest == null");
+ for (KeyVersion keyVersion : deleteKeysRequest.getDeleteKeys().getKeyVersionsList()) {
+ // S3 names the null version with the literal versionId "null", so an
+ // entry names its version one way, never both or neither.
+ if (keyVersion.hasVersionId() == keyVersion.getNullVersion()) {
+ throw new OMException("Entry for key " + keyVersion.getKey()
+ + " has to name one version, by versionId or as the null version", INVALID_REQUEST);
+ }
+ }
if (deleteKeysRequest.getSourceType() == RequestSource.LIFECYCLE && deleteKeysRequest.hasScanState()) {
if (ozoneManager.isAdminAuthorizationEnabled()) {
@@ -110,7 +124,22 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
}
}
- return getOmRequest();
+ // A delete on a versioned bucket creates a marker per key, and a marker is
+ // a version, so it needs an id proposed here on the OM that received the
+ // request. One covers the batch: versionIds only have to increase within a
+ // key, and each key's own current version raises it if needed.
+ return getOmRequest().toBuilder()
+ .setDeleteKeysRequest(deleteKeysRequest.toBuilder()
+ .setDeleteKeys(deleteKeysRequest.getDeleteKeys().toBuilder()
+ // A key or version named twice is deleted once: repeated, it would get a second marker and count
+ // against the quota twice.
+ .clearKeys().addAllKeys(new LinkedHashSet<>(deleteKeysRequest.getDeleteKeys().getKeysList()))
+ .clearKeyVersions()
+ .addAllKeyVersions(deleteKeysRequest.getDeleteKeys().getKeyVersionsList().stream()
+ .map(OMKeysDeleteRequest::canonicalKeyVersion).distinct().collect(Collectors.toList()))
+ .setProposedVersionId(
+ ozoneManager.getVersionIdAllocator().propose())))
+ .build();
}
@Override @SuppressWarnings("methodlength")
@@ -149,6 +178,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredLock = false;
+ // Whether the request may have put an entry into the versionedKeyTable
+ // cache, which decides the failure response below.
+ boolean insertingDeleteMarkers = false;
int indexFailed = 0;
int length = deleteKeys.size();
@@ -157,6 +189,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
.setVolumeName(volumeName).setBucketName(bucketName);
boolean deleteStatus = true;
+ // Keys that stopped being visible to a plain read. A marker written
+ // over a key that was already a marker, or over one the bucket held no
+ // record of, hides nothing that was not hidden already.
+ int visibleKeysRemoved = 0;
long startNanos = Time.monotonicNowNanos();
try {
long startNanosDeleteKeysResolveBucketLatency = Time.monotonicNowNanos();
@@ -183,15 +219,50 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Validate bucket and volume exists or not.
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
String volumeOwner = getVolumeOwner(omMetadataManager, volumeName);
+ OmBucketInfo omBucketInfo =
+ getBucketInfo(omMetadataManager, volumeName, bucketName);
+ // Keys the bucket holds no record of. On a versioned bucket they still
+ // get a delete marker, so they are named rather than skipped.
+ List markerOnlyKeys = new ArrayList<>();
for (indexFailed = 0; indexFailed < length; indexFailed++) {
String keyName = deleteKeyArgs.getKeys(indexFailed);
String objectKey =
omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
+ // Evaluate key ACL before existence, so that a missing key without
+ // DELETE permission is refused rather than being recorded as deleted.
+ try {
+ long startNanosDeleteKeysAclCheckLatency = Time.monotonicNowNanos();
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY,
+ volumeOwner);
+ perfMetrics.setDeleteKeysAclCheckLatencyNs(
+ Time.monotonicNowNanos() - startNanosDeleteKeysAclCheckLatency);
+ } catch (Exception ex) {
+ deleteStatus = false;
+ LOG.error("Acl check failed for Key: {}", objectKey, ex);
+ deleteKeys.remove(keyName);
+ unDeletedKeys.addKeys(keyName);
+ keyToError.put(keyName, new ErrorInfo(OMException.ResultCodes.ACCESS_DENIED.name(), "ACL check failed"));
+ continue;
+ }
+
OmKeyInfo omKeyInfo = getOmKeyInfo(ozoneManager, omMetadataManager,
volumeName, bucketName, keyName);
if (omKeyInfo == null) {
+ // S3 records the delete whether or not the key was there, so a key
+ // with no record takes a delete marker of its own rather than
+ // failing the batch. The single-key delete does the same. A request
+ // that asserts an updateID is excluded: a key with no record has no
+ // updateID that could match. While versioning is suspended the
+ // marker is the key's null version, which may still replace a
+ // noncurrent null version left behind.
+ if (omBucketInfo.hasEverBeenVersioned()
+ && deleteKeyUpdateIDs == null) {
+ markerOnlyKeys.add(keyName);
+ continue;
+ }
deleteStatus = false;
LOG.error("Received a request to delete a Key does not exist {}",
objectKey);
@@ -215,12 +286,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
try {
- // check Acl
- long startNanosDeleteKeysAclCheckLatency = Time.monotonicNowNanos();
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY,
- volumeOwner);
- perfMetrics.setDeleteKeysAclCheckLatencyNs(Time.monotonicNowNanos() - startNanosDeleteKeysAclCheckLatency);
OzoneFileStatus fileStatus = getOzoneKeyStatus(
ozoneManager, omMetadataManager, volumeName, bucketName, keyName);
addKeyToAppropriateList(omKeyInfoList, omKeyInfo, dirList,
@@ -228,26 +293,93 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
deleteKeysInfo.add(omKeyInfo);
} catch (Exception ex) {
deleteStatus = false;
- LOG.error("Acl check failed for Key: {}", objectKey, ex);
+ LOG.error("Failed to prepare Key for deletion: {}", objectKey, ex);
deleteKeys.remove(keyName);
unDeletedKeys.addKeys(keyName);
- keyToError.put(keyName, new ErrorInfo(OMException.ResultCodes.ACCESS_DENIED.name(), "ACL check failed"));
+ keyToError.put(keyName, new ErrorInfo(
+ OMException.ResultCodes.INTERNAL_ERROR.name(), ex.getMessage()));
}
}
- OmBucketInfo omBucketInfo =
- getBucketInfo(omMetadataManager, volumeName, bucketName);
-
Map openKeyInfoMap = new HashMap<>();
- // Mark all keys which can be deleted, in cache as deleted.
- Pair quotaReleasedEmptyKeys =
- markKeysAsDeletedInCache(ozoneManager, trxnLogIndex, omKeyInfoList,
- dirList, omMetadataManager, openKeyInfoMap);
- omBucketInfo.decrUsedBytes(quotaReleasedEmptyKeys.getKey(), true);
- // For empty keyInfos the quota should be released and not added to namespace.
- omBucketInfo.decrUsedNamespace(omKeyInfoList.size() + dirList.size() -
- quotaReleasedEmptyKeys.getValue(), true);
- omBucketInfo.decrUsedNamespace(quotaReleasedEmptyKeys.getValue(), false);
+ // On a bucket that has ever been versioned a delete removes no data:
+ // each key gets a delete marker as its current version, exactly as a
+ // single-key delete does. While versioning is suspended the marker is
+ // the key's null version, which destroys only what held that slot.
+ List markerInsertions = null;
+ if (omBucketInfo.hasEverBeenVersioned()) {
+ insertingDeleteMarkers = true;
+ markerInsertions = insertDeleteMarkers(ozoneManager, omMetadataManager,
+ omBucketInfo, omKeyInfoList, markerOnlyKeys,
+ deleteKeyArgs.getProposedVersionId(), trxnLogIndex);
+ visibleKeysRemoved = (int) omKeyInfoList.stream()
+ .filter(keyInfo -> !keyInfo.isDeleteMarker()).count();
+ } else {
+ // Mark all keys which can be deleted, in cache as deleted.
+ Pair quotaReleasedEmptyKeys =
+ markKeysAsDeletedInCache(ozoneManager, trxnLogIndex, omKeyInfoList,
+ dirList, omMetadataManager, openKeyInfoMap);
+ omBucketInfo.decrUsedBytes(quotaReleasedEmptyKeys.getKey(), true);
+ // For empty keyInfos the quota should be released and not added to namespace.
+ omBucketInfo.decrUsedNamespace(omKeyInfoList.size() + dirList.size() -
+ quotaReleasedEmptyKeys.getValue(), true);
+ omBucketInfo.decrUsedNamespace(quotaReleasedEmptyKeys.getValue(), false);
+ visibleKeysRemoved = deleteKeys.size();
+ }
+
+ // Entries naming a version permanently delete it, as a DeleteKey with a
+ // versionId does. They run after the markers, so a key named both ways
+ // is marked first. An entry that fails is reported on its own.
+ List versionDeletes = new ArrayList<>();
+ List versionErrors = new ArrayList<>();
+ List deletedVersions = new ArrayList<>();
+ for (KeyVersion keyVersion : deleteKeyArgs.getKeyVersionsList()) {
+ String keyName = keyVersion.getKey();
+ try {
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY, volumeOwner);
+ } catch (Exception ex) {
+ LOG.error("Acl check failed for Key: {}", keyName, ex);
+ addVersionError(versionErrors, unDeletedKeys, keyVersion,
+ OMException.ResultCodes.ACCESS_DENIED, "ACL check failed");
+ continue;
+ }
+ if (!omBucketInfo.hasEverBeenVersioned()) {
+ // as a single delete reports it
+ addVersionError(versionErrors, unDeletedKeys, keyVersion,
+ OMException.ResultCodes.KEY_NOT_FOUND, "Bucket does not have S3 versioning enabled");
+ continue;
+ }
+ OmKeyInfo currentVersion = getOmKeyInfo(ozoneManager, omMetadataManager, volumeName, bucketName, keyName);
+ try {
+ // The batch writes these out through its own response, so the
+ // OMResponse each one carries is never sent.
+ versionDeletes.add(deleteVersion(ozoneManager, omMetadataManager, omBucketInfo,
+ currentVersion, volumeName, bucketName, keyName, keyVersion.getVersionId(),
+ keyVersion.getNullVersion(), trxnLogIndex, OmResponseUtil.getOMResponseBuilder(getOmRequest()),
+ null));
+ } catch (OMException ex) {
+ if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
+ throw ex;
+ }
+ addVersionError(versionErrors, unDeletedKeys, keyVersion,
+ OMException.ResultCodes.KEY_NOT_FOUND, ex.getMessage());
+ continue;
+ }
+ deletedVersions.add(keyName + "=" + (keyVersion.getNullVersion() ? "null" : keyVersion.getVersionId()));
+ // As in a single delete: a visible key goes away only when its current
+ // version is deleted with nothing left to promote.
+ if (currentVersion != null && !currentVersion.isDeleteMarker()
+ && getOmKeyInfo(ozoneManager, omMetadataManager, volumeName, bucketName, keyName) == null) {
+ visibleKeysRemoved++;
+ }
+ }
+ if (!versionErrors.isEmpty()) {
+ deleteStatus = false;
+ }
+ if (!deletedVersions.isEmpty()) {
+ auditMap.put(OzoneConsts.DELETED_VERSION_ID, String.join(",", deletedVersions));
+ }
OmLifecycleScanState state = null;
if (sourceType == RequestSource.LIFECYCLE && deleteKeyRequest.hasScanState()) {
@@ -259,9 +391,19 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
final long volumeId = omMetadataManager.getVolumeId(volumeName);
- omClientResponse =
- getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse,
- unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap, state);
+ if (markerInsertions != null) {
+ omResponse.setDeleteKeysResponse(DeleteKeysResponse.newBuilder()
+ .setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys)
+ .addAllErrors(toDeleteKeyErrors(keyToError, versionErrors)))
+ .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus);
+ omClientResponse = new OMKeysDeleteMarkerResponse(omResponse.build(),
+ markerInsertions, versionDeletes, omBucketInfo.copyObject(), state);
+ } else {
+ omClientResponse =
+ getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse,
+ unDeletedKeys, keyToError, versionErrors, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap,
+ state);
+ }
result = Result.SUCCESS;
long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos();
@@ -280,12 +422,20 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
keyToError.put(deleteKeyArgs.getKeys(i), new ErrorInfo(OMException.ResultCodes.INTERNAL_ERROR.name(),
ex.getMessage()));
}
+ unDeletedKeys.clearKeyVersions().addAllKeyVersions(deleteKeyArgs.getKeyVersionsList());
omResponse.setDeleteKeysResponse(
DeleteKeysResponse.newBuilder().setStatus(false)
.setUnDeletedKeys(unDeletedKeys).build()).build();
- omClientResponse =
- new OMKeysDeleteResponse(omResponse.build(), getBucketLayout());
+ // The failure response has to declare the same tables as the successful
+ // one: the double buffer cleans up the table cache from the response's
+ // CleanupTableInfo, so a marker insertion that failed part way through
+ // the batch would otherwise leave a versionedKeyTable entry behind that
+ // is in no DB and is never cleaned up.
+ omClientResponse = insertingDeleteMarkers
+ ? new OMKeysDeleteMarkerResponse(omResponse.build(),
+ getBucketLayout())
+ : new OMKeysDeleteResponse(omResponse.build(), getBucketLayout());
long endNanosDeleteKeyFailureLatencyNs = Time.monotonicNowNanos();
perfMetrics.setDeleteKeyFailureLatencyNs(endNanosDeleteKeyFailureLatencyNs - startNanos);
} finally {
@@ -319,7 +469,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
omMetrics.incNumKeyDeletes(deleteKeys.size());
omMetrics.incNumKeyDeleteFails(unDeletedKeys.getKeysList().size());
- omMetrics.decNumKeys(deleteKeys.size());
+ omMetrics.decNumKeys(visibleKeysRemoved);
if (LOG.isDebugEnabled()) {
LOG.debug("Keys delete success. Volume:{}, Bucket:{}, Keys:{}, sourceType:{}",
volumeName, bucketName, auditMap.get(DELETED_KEYS_LIST), sourceType);
@@ -359,21 +509,53 @@ protected OzoneFileStatus getOzoneKeyStatus(
return null;
}
+ /**
+ * The entry with only the fields that name its version, so that two entries naming the same version compare equal
+ * whatever else they set.
+ */
+ private static KeyVersion canonicalKeyVersion(KeyVersion keyVersion) {
+ KeyVersion.Builder canonical = KeyVersion.newBuilder().setKey(keyVersion.getKey());
+ return (keyVersion.hasVersionId()
+ ? canonical.setVersionId(keyVersion.getVersionId())
+ : canonical.setNullVersion(true)).build();
+ }
+
+ /** Records an entry naming a version that the batch could not delete. */
+ private static void addVersionError(List errors, DeleteKeyArgs.Builder unDeletedKeys,
+ KeyVersion keyVersion, OMException.ResultCodes code, String message) {
+ unDeletedKeys.addKeyVersions(keyVersion);
+ DeleteKeyError.Builder error = DeleteKeyError.newBuilder()
+ .setKey(keyVersion.getKey()).setErrorCode(code.name()).setErrorMsg(message);
+ if (keyVersion.hasVersionId()) {
+ error.setVersionId(keyVersion.getVersionId());
+ } else {
+ error.setNullVersion(true);
+ }
+ errors.add(error.build());
+ }
+
+ /** Every error of the batch: the entries naming a version and the keys. */
+ private static List toDeleteKeyErrors(Map keyToErrors,
+ List versionErrors) {
+ List deleteKeyErrors = new ArrayList<>(versionErrors);
+ for (Map.Entry key : keyToErrors.entrySet()) {
+ deleteKeyErrors.add(DeleteKeyError.newBuilder().setKey(key.getKey())
+ .setErrorCode(key.getValue().getCode()).setErrorMsg(key.getValue().getMessage()).build());
+ }
+ return deleteKeyErrors;
+ }
+
@Nonnull
@SuppressWarnings("parameternumber")
protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
List omKeyInfoList, List dirList,
OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
- Map keyToErrors,
+ Map keyToErrors, List versionErrors,
boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap,
OmLifecycleScanState scanState) {
OMClientResponse omClientResponse;
- List deleteKeyErrors = new ArrayList<>();
- for (Map.Entry key : keyToErrors.entrySet()) {
- deleteKeyErrors.add(OzoneManagerProtocolProtos.DeleteKeyError.newBuilder().setKey(key.getKey())
- .setErrorCode(key.getValue().getCode()).setErrorMsg(key.getValue().getMessage()).build());
- }
+ List deleteKeyErrors = toDeleteKeyErrors(keyToErrors, versionErrors);
omClientResponse = new OMKeysDeleteResponse(omResponse
.setDeleteKeysResponse(
DeleteKeysResponse.newBuilder().setStatus(deleteStatus)
@@ -384,6 +566,54 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
return omClientResponse;
}
+ /**
+ * Inserts a delete marker for every key that survived validation, reusing
+ * the implementation a single-key delete uses so the two cannot drift.
+ *
+ * @return what each insertion changed, for the response to write out
+ */
+ private List insertDeleteMarkers(
+ OzoneManager ozoneManager, OMMetadataManager omMetadataManager,
+ OmBucketInfo omBucketInfo, List omKeyInfoList,
+ List markerOnlyKeys, long proposedVersionId, long trxnLogIndex)
+ throws IOException {
+
+ List insertions = new ArrayList<>();
+ // Applied to the bucket only once every marker is in, so that a batch
+ // failing on a later key leaves the cached bucket as it found it.
+ long pendingNamespace = 0;
+ long pendingReleasedBytes = 0;
+ for (OmKeyInfo currentVersion : omKeyInfoList) {
+ String objectKey = omMetadataManager.getOzoneKey(
+ currentVersion.getVolumeName(), currentVersion.getBucketName(),
+ currentVersion.getKeyName());
+ DeleteMarkerInsertion insertion = insertDeleteMarker(ozoneManager,
+ omMetadataManager, omBucketInfo, currentVersion, objectKey,
+ currentVersion.getKeyName(), proposedVersionId,
+ Time.now(), trxnLogIndex, pendingNamespace);
+ pendingNamespace += insertion.getAddedNamespace();
+ pendingReleasedBytes += insertion.getReleasedBytes();
+ insertions.add(insertion);
+ }
+ // A key the bucket holds no record of supersedes nothing, so the marker
+ // is built without a current version to inherit from.
+ for (String keyName : markerOnlyKeys) {
+ String objectKey = omMetadataManager.getOzoneKey(
+ omBucketInfo.getVolumeName(), omBucketInfo.getBucketName(), keyName);
+ DeleteMarkerInsertion insertion = insertDeleteMarker(ozoneManager,
+ omMetadataManager, omBucketInfo, null, objectKey, keyName,
+ proposedVersionId, Time.now(), trxnLogIndex, pendingNamespace);
+ pendingNamespace += insertion.getAddedNamespace();
+ pendingReleasedBytes += insertion.getReleasedBytes();
+ insertions.add(insertion);
+ }
+ omBucketInfo.incrUsedNamespace(pendingNamespace);
+ if (pendingReleasedBytes > 0) {
+ omBucketInfo.decrUsedBytes(pendingReleasedBytes, true);
+ }
+ return insertions;
+ }
+
protected Pair markKeysAsDeletedInCache(OzoneManager ozoneManager,
long trxnLogIndex, List omKeyInfoList, List dirList,
OMMetadataManager omMetadataManager, Map openKeyInfoMap)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMObjectVersionsReclaimRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMObjectVersionsReclaimRequest.java
new file mode 100644
index 000000000000..19478535c3be
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMObjectVersionsReclaimRequest.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.OMSystemAction;
+import org.apache.hadoop.ozone.om.NoncurrentVersions;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMObjectVersionsReclaimResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionRecord;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionsBucket;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ReclaimObjectVersionsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ReclaimObjectVersionsResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles the reclamation the lifecycle scan selects on a versioned bucket:
+ * noncurrent versions a NoncurrentVersionExpiration rule expires, which leave
+ * the versionedKeyTable with their blocks queued in the deletedTable, and
+ * expired delete markers, which leave the keyTable so the key disappears with
+ * them. Nothing here reclaims blocks directly.
+ *
+ * The scan selects from the DB as of its own pass, so a record may already
+ * be gone, or no longer eligible, by the time this request applies - a version
+ * permanently deleted or promoted into the keyTable, a marker superseded by a
+ * write. Every condition the scan checked is checked again here under the
+ * bucket lock, and anything that no longer holds is skipped; a later pass
+ * reselects if it still applies.
+ */
+public class OMObjectVersionsReclaimRequest extends OMKeyRequest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMObjectVersionsReclaimRequest.class);
+
+ private static final AuditLogger AUDIT =
+ new AuditLogger(AuditLoggerType.OMSYSTEMLOGGER);
+ private static final String AUDIT_PARAM_NUM_VERSIONS =
+ "numObjectVersionsReclaimed";
+
+ public OMObjectVersionsReclaimRequest(OMRequest omRequest) {
+ // S3 object versioning is supported on OBJECT_STORE buckets only, so every
+ // version this request touches lives in the OBJECT_STORE keyTable.
+ super(omRequest, BucketLayout.OBJECT_STORE);
+ }
+
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ ExecutionContext context) {
+ final long trxnLogIndex = context.getIndex();
+
+ ReclaimObjectVersionsRequest reclaimRequest =
+ getOmRequest().getReclaimObjectVersionsRequest();
+ List versionsPerBucket =
+ reclaimRequest.getVersionsPerBucketList();
+
+ long numSubmitted = 0;
+ for (ObjectVersionsBucket bucket : versionsPerBucket) {
+ numSubmitted += bucket.getVersionsCount() + bucket.getMarkersCount();
+ }
+
+ OMResponse.Builder omResponse =
+ OmResponseUtil.getOMResponseBuilder(getOmRequest());
+ omResponse.setReclaimObjectVersionsResponse(
+ ReclaimObjectVersionsResponse.newBuilder());
+
+ OMClientResponse omClientResponse = null;
+ List reclaimedVersionKeys = new ArrayList<>();
+ List reclaimedMarkerKeys = new ArrayList<>();
+ Map keysToDelete = new HashMap<>();
+ List updatedBuckets = new ArrayList<>();
+ Map auditParams = new LinkedHashMap<>();
+ try {
+ for (ObjectVersionsBucket bucket : versionsPerBucket) {
+ reclaimBucketVersions(ozoneManager, trxnLogIndex, bucket,
+ reclaimedVersionKeys, reclaimedMarkerKeys, keysToDelete,
+ updatedBuckets);
+ }
+
+ omClientResponse = new OMObjectVersionsReclaimResponse(
+ omResponse.build(), reclaimedVersionKeys, reclaimedMarkerKeys,
+ keysToDelete, updatedBuckets);
+
+ int reclaimed = reclaimedVersionKeys.size() + reclaimedMarkerKeys.size();
+ ozoneManager.getDeletionMetrics()
+ .incrNumObjectVersionsReclaimed(reclaimed);
+ auditParams.put(AUDIT_PARAM_NUM_VERSIONS, String.valueOf(reclaimed));
+ AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(
+ OMSystemAction.OBJECT_VERSION_CLEANUP, auditParams));
+ LOG.debug("Reclaimed {} object versions and {} expired delete markers "
+ + "out of {} submitted.", reclaimedVersionKeys.size(),
+ reclaimedMarkerKeys.size(), numSubmitted);
+ } catch (IOException ex) {
+ AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(
+ OMSystemAction.OBJECT_VERSION_CLEANUP, auditParams, ex));
+ LOG.error("Failed to reclaim {} submitted records.", numSubmitted, ex);
+ omClientResponse = new OMObjectVersionsReclaimResponse(
+ createErrorOMResponse(omResponse, ex));
+ } finally {
+ if (omClientResponse != null) {
+ omClientResponse.setOmLockDetails(getOmLockDetails());
+ }
+ }
+
+ return omClientResponse;
+ }
+
+ /** Whether the key has any version left in the versionedKeyTable. */
+ private boolean hasNoncurrentVersion(OMMetadataManager omMetadataManager,
+ OmKeyInfo marker) throws IOException {
+ return NoncurrentVersions.any(omMetadataManager, marker.getVolumeName(),
+ marker.getBucketName(), marker.getKeyName());
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private void reclaimBucketVersions(OzoneManager ozoneManager,
+ long trxnLogIndex, ObjectVersionsBucket versionsBucket,
+ List reclaimedVersionKeys, List reclaimedMarkerKeys,
+ Map keysToDelete,
+ List updatedBuckets) throws IOException {
+
+ String volumeName = versionsBucket.getVolumeName();
+ String bucketName = versionsBucket.getBucketName();
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+
+ boolean acquiredLock = false;
+ try {
+ mergeOmLockDetails(omMetadataManager.getLock()
+ .acquireWriteLock(BUCKET_LOCK, volumeName, bucketName));
+ acquiredLock = getOmLockDetails().isLockAcquired();
+
+ OmBucketInfo omBucketInfo =
+ getBucketInfo(omMetadataManager, volumeName, bucketName);
+ if (omBucketInfo == null) {
+ LOG.debug("Bucket {}/{} no longer exists, skipping the {} object "
+ + "versions and {} delete markers submitted for it.", volumeName,
+ bucketName, versionsBucket.getVersionsCount(),
+ versionsBucket.getMarkersCount());
+ return;
+ }
+
+ boolean reclaimedAny = false;
+ for (ObjectVersionRecord submitted : versionsBucket.getVersionsList()) {
+ String versionKey = submitted.getDbKey();
+ OmKeyInfo version =
+ omMetadataManager.getVersionedKeyTable().get(versionKey);
+ if (version == null) {
+ // Already reclaimed, permanently deleted, or promoted into the
+ // keyTable since the service selected it.
+ continue;
+ }
+ if (version.getUpdateID() != submitted.getUpdateId()) {
+ // A different record sits where the scan looked. A versionId is
+ // allocated against the key's current version alone, so one that was
+ // permanently deleted can be handed out again, putting a new version
+ // at the dbKey the scan selected.
+ LOG.debug("Version {} was written again since the scan selected it, "
+ + "skipping reclamation.", versionKey);
+ continue;
+ }
+
+ version = version.toBuilder().setUpdateID(trxnLogIndex).build();
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(versionKey), CacheValue.get(trxnLogIndex));
+ reclaimedVersionKeys.add(versionKey);
+
+ // A delete marker holds no blocks, so it releases namespace but no
+ // space, and there is nothing to reclaim for it: an empty record is
+ // not queued in the deletedTable at all, as every other delete path
+ // does through AbstractOMKeyDeleteResponse.
+ boolean isVersionNonEmpty = !OmKeyInfo.isKeyEmpty(version);
+ if (isVersionNonEmpty) {
+ // Versions of one key share a deletedTable entry: it holds a
+ // RepeatedOmKeyInfo list that KeyDeletingService evaluates one
+ // record at a time.
+ String ozoneKey = omMetadataManager.getOzoneKey(volumeName,
+ bucketName, version.getKeyName());
+ addKeyInfoToDeleteMap(ozoneManager, trxnLogIndex, ozoneKey,
+ omBucketInfo.getObjectID(),
+ version.withCommittedKeyDeletedFlag(true), keysToDelete);
+ }
+ omBucketInfo.decrUsedBytes(sumBlockLengths(version), isVersionNonEmpty);
+ omBucketInfo.decrUsedNamespace(1L, isVersionNonEmpty);
+ reclaimedAny = true;
+ }
+
+ for (ObjectVersionRecord submitted : versionsBucket.getMarkersList()) {
+ String markerKey = submitted.getDbKey();
+ OmKeyInfo marker =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(markerKey);
+ if (marker == null) {
+ continue;
+ }
+ // A write since the scan supersedes the marker: the key's current
+ // version is then a real object, and there is nothing expired here.
+ if (!marker.isDeleteMarker()) {
+ continue;
+ }
+ // A marker's dbKey is the plain key name, so it names whichever
+ // version of the key is current rather than the record the scan read.
+ // Deleting the key and deleting it again leaves another marker there,
+ // which no other check here can tell from the expired one.
+ if (marker.getUpdateID() != submitted.getUpdateId()) {
+ LOG.debug("The delete marker of {} was replaced since the scan "
+ + "selected it, skipping reclamation.", markerKey);
+ continue;
+ }
+ // Re-checked here and not only in the scan: removing the marker while
+ // a noncurrent version survives would make that version current again,
+ // resurrecting an object the user deleted.
+ if (hasNoncurrentVersion(omMetadataManager, marker)) {
+ continue;
+ }
+
+ omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(
+ new CacheKey<>(markerKey), CacheValue.get(trxnLogIndex));
+ reclaimedMarkerKeys.add(markerKey);
+
+ // The marker holds no blocks, so it never reaches the deletedTable; it
+ // does hold a namespace slot of its own.
+ omBucketInfo.decrUsedNamespace(1L, false);
+ reclaimedAny = true;
+ }
+
+ if (reclaimedAny) {
+ updatedBuckets.add(omBucketInfo.copyObject());
+ }
+ } finally {
+ if (acquiredLock) {
+ mergeOmLockDetails(omMetadataManager.getLock()
+ .releaseWriteLock(BUCKET_LOCK, volumeName, bucketName));
+ }
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
index 51de2dd0753e..daf645930456 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
@@ -165,11 +165,11 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
List omKeyInfoList, List dirList,
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
- Map keyToErrors,
+ Map keyToErrors, List versionErrors,
boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap, OmLifecycleScanState state) {
OMClientResponse omClientResponse;
- List deleteKeyErrors = new ArrayList<>();
+ List deleteKeyErrors = new ArrayList<>(versionErrors);
for (Map.Entry key : keyToErrors.entrySet()) {
deleteKeyErrors.add(OzoneManagerProtocolProtos.DeleteKeyError.newBuilder()
.setKey(key.getKey()).setErrorCode(key.getValue().getCode())
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index e11f2210ebcb..0d761af3855b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -34,6 +34,7 @@
import java.util.function.BiFunction;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -55,6 +56,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
@@ -125,7 +127,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
keyPath, getBucketLayout());
KeyArgs newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now())
- .setKeyName(keyPath).build();
+ .setKeyName(keyPath)
+ // Completing an upload creates a version like any other write, so
+ // its id is proposed here on the OM that received the request.
+ // Unused when the bucket has never been versioned.
+ .setProposedVersionId(
+ ozoneManager.getVersionIdAllocator().propose())
+ .build();
KeyArgs resolvedArgs = resolveBucketAndCheckKeyAcls(newKeyArgs,
ozoneManager, ACLType.WRITE);
@@ -338,7 +346,26 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
OmKeyInfo keyToDelete =
omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
boolean isNamespaceUpdate = false;
- if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
+ // Completing a multipart upload creates a version like any other
+ // write: the version it supersedes is kept instead of reclaimed,
+ // except for the null version, which a suspended write replaces.
+ final boolean supersededVersionRetained = keyToDelete != null
+ && omBucketInfo.hasEverBeenVersioned()
+ && (omBucketInfo.isS3VersioningEnabled()
+ || !keyToDelete.isNullVersionRecord());
+ if (omBucketInfo.hasEverBeenVersioned()) {
+ omKeyInfo = omKeyInfo.toBuilder()
+ .setVersionId(ozoneManager.getVersionIdAllocator().allocate(
+ keyArgs.getProposedVersionId(), keyToDelete))
+ .setNullVersion(omBucketInfo.isS3VersioningSuspended())
+ .build();
+ auditMap.put(OzoneConsts.VERSION_ID,
+ String.valueOf(omKeyInfo.getVersionId()));
+ auditMap.put(OzoneConsts.NULL_VERSION,
+ String.valueOf(omKeyInfo.isNullVersion()));
+ }
+ if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()
+ && !supersededVersionRetained) {
RepeatedOmKeyInfo oldKeyVersionsToDelete = getOldVersionsToCleanUp(
keyToDelete, omBucketInfo.getObjectID(), trxnLogIndex);
allKeyInfoToRemove.addAll(oldKeyVersionsToDelete.getOmKeyInfoList());
@@ -349,6 +376,49 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
isNamespaceUpdate = true;
}
+ // The superseded version becomes a noncurrent version. Without this the
+ // record would be dropped from the keyTable, kept out of the
+ // deletedTable by the check above, and leak.
+ String dbVersionedKey = null;
+ OmKeyInfo versionedKeyInfo = null;
+ if (supersededVersionRetained) {
+ // This upload is what supersedes it, so this is when it stopped
+ // being current.
+ OmKeyInfo.Builder versionedBuilder = keyToDelete.toBuilder()
+ .setNoncurrentTime(keyArgs.getModificationTime());
+ if (keyToDelete.getVersionId() == null) {
+ versionedBuilder
+ .setVersionId(VersionIdGenerator.UNSET_VERSION_ID)
+ .setNullVersion(true);
+ }
+ versionedKeyInfo = versionedBuilder.build();
+ dbVersionedKey = omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionedKeyInfo.getVersionId());
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ dbVersionedKey, versionedKeyInfo, trxnLogIndex);
+ auditMap.put(OzoneConsts.SUPERSEDED_VERSION_ID,
+ String.valueOf(versionedKeyInfo.getVersionId()));
+ }
+
+ // A suspended write replaces the key's null version wherever it is.
+ String replacedNullVersionKey = null;
+ if (omBucketInfo.isS3VersioningSuspended()
+ && supersededVersionRetained) {
+ Pair nullVersion = getNoncurrentNullVersion(
+ ozoneManager, omMetadataManager, volumeName, bucketName,
+ keyName);
+ if (nullVersion != null) {
+ replacedNullVersionKey = nullVersion.getKey();
+ allKeyInfoToRemove.add(nullVersion.getValue()
+ .withCommittedKeyDeletedFlag(true));
+ usedBytesDiff -= nullVersion.getValue().getReplicatedSize();
+ omBucketInfo.decrUsedNamespace(1L, true);
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(replacedNullVersionKey),
+ CacheValue.get(trxnLogIndex));
+ }
+ }
+
String dbBucketKey = omMetadataManager.getBucketKey(
omBucketInfo.getVolumeName(), omBucketInfo.getBucketName());
if (usedBytesDiff != 0) {
@@ -373,10 +443,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
long volumeId = omMetadataManager.getVolumeId(volumeName);
long bucketId = omMetadataManager.getBucketId(volumeName, bucketName);
omClientResponse =
- getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey,
+ ((S3MultipartUploadCompleteResponse) getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey,
omKeyInfo, allKeyInfoToRemove, omBucketInfo,
volumeId, bucketId, missingParentInfos, multipartKeyInfo,
- multipartPartKeysToDelete);
+ multipartPartKeysToDelete))
+ .withVersionedKey(dbVersionedKey, versionedKeyInfo,
+ replacedNullVersionKey);
result = Result.SUCCESS;
} else {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
index ae65be2ad6e8..86dd085046f5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf;
import static org.apache.hadoop.hdds.HddsUtils.toProtobuf;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.SNAPSHOT_LOCK;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT;
@@ -174,6 +175,24 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Snapshot referenced size should be bucket's used bytes
OmBucketInfo omBucketInfo =
getBucketInfo(omMetadataManager, volumeName, bucketName);
+
+ // volumeName and bucketName were resolved to the source bucket in
+ // preExecute, so a linked bucket is judged by what it points at. The
+ // check runs here rather than in preExecute so that it and the versioning
+ // transition check in OMBucketSetPropertyRequest are ordered by Ratis:
+ // whichever applies second sees the state the first one committed.
+ if (omBucketInfo.hasEverBeenVersioned()
+ && !ozoneManager.isSnapshotVersioningCoexistenceAllowed()) {
+ throw new OMException("Cannot create a snapshot of bucket "
+ + bucketName + ": its object versioning status is "
+ + omBucketInfo.getVersioningStatus() + ". Snapshots and S3 object"
+ + " versioning cannot be used on the same bucket until"
+ + " snapshot-aware version reclamation is available; without it a"
+ + " noncurrent version reclaimed from the active object store"
+ + " would take blocks a snapshot still references.",
+ NOT_SUPPORTED_OPERATION);
+ }
+
snapshotInfo.setReferencedReplicatedSize(omBucketInfo.getUsedBytes());
// Snapshot referenced size in this case is an *estimate* inferred from
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java
index 425c4f63ac5e..ed17907b4fa0 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java
@@ -21,6 +21,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
import com.google.common.annotations.VisibleForTesting;
import jakarta.annotation.Nonnull;
@@ -39,7 +40,7 @@
* Response for CommitKey request.
*/
@CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, KEY_TABLE, DELETED_TABLE,
- BUCKET_TABLE})
+ BUCKET_TABLE, VERSIONED_KEY_TABLE})
public class OMKeyCommitResponse extends OmKeyResponse {
private OmKeyInfo omKeyInfo;
@@ -51,6 +52,9 @@ public class OMKeyCommitResponse extends OmKeyResponse {
private OmKeyInfo newOpenKeyInfo;
private OmKeyInfo openKeyToUpdate;
private String openKeyNameToUpdate;
+ private String versionedKeyName;
+ private OmKeyInfo versionedKeyInfo;
+ private String replacedNullVersionKey;
@SuppressWarnings("checkstyle:ParameterNumber")
public OMKeyCommitResponse(
@@ -82,6 +86,27 @@ public OMKeyCommitResponse(@Nonnull OMResponse omResponse, @Nonnull
checkStatusNotOK();
}
+ /**
+ * The version this commit overwrote, to be kept in the versionedKeyTable as
+ * a noncurrent version. Null for buckets without S3-compatible versioning.
+ */
+ public OMKeyCommitResponse withVersionedKey(String dbVersionedKey,
+ OmKeyInfo keyInfo) {
+ this.versionedKeyName = dbVersionedKey;
+ this.versionedKeyInfo = keyInfo;
+ return this;
+ }
+
+ /**
+ * The noncurrent null version this commit replaced, to be removed from the
+ * versionedKeyTable. Null unless a suspended write replaced a null version
+ * that was not the current one.
+ */
+ public OMKeyCommitResponse withReplacedNullVersion(String dbVersionedKey) {
+ this.replacedNullVersionKey = dbVersionedKey;
+ return this;
+ }
+
@Override
public void addToDBBatch(OMMetadataManager omMetadataManager,
BatchOperation batchOperation) throws IOException {
@@ -98,6 +123,16 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation, ozoneKeyName, omKeyInfo);
+ if (versionedKeyInfo != null) {
+ omMetadataManager.getVersionedKeyTable()
+ .putWithBatch(batchOperation, versionedKeyName, versionedKeyInfo);
+ }
+
+ if (replacedNullVersionKey != null) {
+ omMetadataManager.getVersionedKeyTable()
+ .deleteWithBatch(batchOperation, replacedNullVersionKey);
+ }
+
updateDeletedTable(omMetadataManager, batchOperation);
handleOpenKeyToUpdate(omMetadataManager, batchOperation);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteMarkerResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteMarkerResponse.java
new file mode 100644
index 000000000000..f5c8af46d21e
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteMarkerResponse.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
+
+import com.google.common.annotations.VisibleForTesting;
+import jakarta.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Response for a DeleteKey request on a bucket with S3-compatible versioning
+ * enabled: no data is removed. A delete marker becomes the current version in
+ * the keyTable, and the version it supersedes (if the key existed) moves to
+ * the versionedKeyTable.
+ */
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, VERSIONED_KEY_TABLE, BUCKET_TABLE,
+ DELETED_TABLE})
+public class OMKeyDeleteMarkerResponse extends OmKeyResponse {
+
+ private OmKeyInfo deleteMarker;
+ private String ozoneKeyName;
+ private String movedVersionedKeyName;
+ private OmKeyInfo movedVersionedKeyInfo;
+ private OmBucketInfo omBucketInfo;
+ private String replacedNullVersionKey;
+ private Map keysToDelete;
+
+ /**
+ * @param movedVersionedKeyName the versionedKeyTable entry the superseded
+ * version moves to, or null when the marker replaced it instead
+ * @param replacedNullVersionKey the versionedKeyTable entry of the null
+ * version the marker replaced, when versioning is suspended and it had
+ * one
+ * @param keysToDelete the replaced null version's blocks, queued for
+ * reclamation
+ */
+ @SuppressWarnings("parameternumber")
+ public OMKeyDeleteMarkerResponse(@Nonnull OMResponse omResponse,
+ @Nonnull OmKeyInfo deleteMarker, @Nonnull String ozoneKeyName,
+ String movedVersionedKeyName, OmKeyInfo movedVersionedKeyInfo,
+ @Nonnull OmBucketInfo omBucketInfo, String replacedNullVersionKey,
+ Map keysToDelete) {
+ super(omResponse, omBucketInfo.getBucketLayout());
+ this.deleteMarker = deleteMarker;
+ this.ozoneKeyName = ozoneKeyName;
+ this.movedVersionedKeyName = movedVersionedKeyName;
+ this.movedVersionedKeyInfo = movedVersionedKeyInfo;
+ this.omBucketInfo = omBucketInfo;
+ this.replacedNullVersionKey = replacedNullVersionKey;
+ this.keysToDelete = keysToDelete;
+ }
+
+ /**
+ * For when the request is not successful.
+ * For a successful request, the other constructor should be used.
+ */
+ public OMKeyDeleteMarkerResponse(@Nonnull OMResponse omResponse,
+ @Nonnull BucketLayout bucketLayout) {
+ super(omResponse, bucketLayout);
+ checkStatusNotOK();
+ }
+
+ @VisibleForTesting
+ public Map getKeysToDelete() {
+ return keysToDelete;
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .putWithBatch(batchOperation, ozoneKeyName, deleteMarker);
+
+ if (movedVersionedKeyInfo != null) {
+ omMetadataManager.getVersionedKeyTable().putWithBatch(batchOperation,
+ movedVersionedKeyName, movedVersionedKeyInfo);
+ }
+
+ if (replacedNullVersionKey != null) {
+ omMetadataManager.getVersionedKeyTable()
+ .deleteWithBatch(batchOperation, replacedNullVersionKey);
+ }
+
+ if (keysToDelete != null) {
+ for (Map.Entry entry
+ : keysToDelete.entrySet()) {
+ omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
+ entry.getKey(), entry.getValue());
+ }
+ }
+
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
+ omBucketInfo.getBucketName()), omBucketInfo);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyVersionDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyVersionDeleteResponse.java
new file mode 100644
index 000000000000..fc02b7585fc8
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyVersionDeleteResponse.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
+
+import jakarta.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Response for {@code DELETE ?versionId=}: the version leaves the table that
+ * held it and its blocks go to the deletedTable, which is the single path
+ * through which version blocks are reclaimed. Removing the current version
+ * promotes the newest remaining one into the keyTable in the same batch, so the
+ * key never lacks a current version; if none remains the key disappears.
+ */
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, VERSIONED_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE})
+public class OMKeyVersionDeleteResponse extends AbstractOMKeyDeleteResponse {
+
+ private final OmKeyInfo deletedVersion;
+ private final String deletedKeyName;
+ private final boolean deletedCurrent;
+ private final String promotedKeyName;
+ private final OmKeyInfo promoted;
+ private final OmBucketInfo omBucketInfo;
+ private final String deletedTableKey;
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public OMKeyVersionDeleteResponse(@Nonnull OMResponse omResponse,
+ @Nonnull OmKeyInfo deletedVersion, @Nonnull String deletedKeyName,
+ boolean deletedCurrent, String promotedKeyName, OmKeyInfo promoted,
+ @Nonnull OmBucketInfo omBucketInfo, @Nonnull String deletedTableKey) {
+ super(omResponse, omBucketInfo.getBucketLayout());
+ this.deletedVersion = deletedVersion;
+ this.deletedKeyName = deletedKeyName;
+ this.deletedCurrent = deletedCurrent;
+ this.promotedKeyName = promotedKeyName;
+ this.promoted = promoted;
+ this.omBucketInfo = omBucketInfo;
+ this.deletedTableKey = deletedTableKey;
+ }
+
+ /**
+ * For when the request is not successful.
+ * For a successful request, the other constructor should be used.
+ */
+ public OMKeyVersionDeleteResponse(@Nonnull OMResponse omResponse,
+ @Nonnull BucketLayout bucketLayout) {
+ super(omResponse, bucketLayout);
+ this.deletedVersion = null;
+ this.deletedKeyName = null;
+ this.deletedCurrent = false;
+ this.promotedKeyName = null;
+ this.promoted = null;
+ this.omBucketInfo = null;
+ this.deletedTableKey = null;
+ checkStatusNotOK();
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ // The version leaves whichever table held it and its blocks go to the
+ // deletedTable; when it was the current version the successor takes its
+ // place in the keyTable, so the delete and the promotion land in one batch.
+ addDeletionToBatch(omMetadataManager, batchOperation,
+ deletedCurrent ? omMetadataManager.getKeyTable(getBucketLayout())
+ : omMetadataManager.getVersionedKeyTable(),
+ deletedKeyName, deletedTableKey, deletedVersion,
+ omBucketInfo.getObjectID(), true);
+ addPromotionToBatch(omMetadataManager, batchOperation);
+
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
+ omBucketInfo.getBucketName()), omBucketInfo);
+ }
+
+ /**
+ * Adds this delete to the batch of a DeleteKeys request, which writes the
+ * bucket itself. Versions of one key deleted in one transaction share a
+ * deletedTable key, so their blocks are gathered in {@code deletedBlocks} for
+ * the caller to put once: a put per version would keep only the last one's.
+ */
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation, Map deletedBlocks) throws IOException {
+ (deletedCurrent ? omMetadataManager.getKeyTable(getBucketLayout()) : omMetadataManager.getVersionedKeyTable())
+ .deleteWithBatch(batchOperation, deletedKeyName);
+ if (!OmKeyInfo.isKeyEmpty(deletedVersion)) {
+ RepeatedOmKeyInfo blocks = OmUtils.prepareKeyForDelete(omBucketInfo.getObjectID(),
+ deletedVersion.withCommittedKeyDeletedFlag(true), deletedVersion.getUpdateID());
+ RepeatedOmKeyInfo gathered = deletedBlocks.putIfAbsent(deletedTableKey, blocks);
+ if (gathered != null) {
+ blocks.getOmKeyInfoList().forEach(gathered::addOmKeyInfo);
+ }
+ }
+ addPromotionToBatch(omMetadataManager, batchOperation);
+ }
+
+ private void addPromotionToBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ if (promoted != null) {
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .putWithBatch(batchOperation, deletedKeyName, promoted);
+ omMetadataManager.getVersionedKeyTable()
+ .deleteWithBatch(batchOperation, promotedKeyName);
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteMarkerResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteMarkerResponse.java
new file mode 100644
index 000000000000..b7080cb06848
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteMarkerResponse.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE;
+
+import jakarta.annotation.Nonnull;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.key.DeleteMarkerInsertion;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Response for a batch DeleteKeys request on a bucket that has ever been
+ * versioned: no version the key still needs is removed. Each key gets a delete
+ * marker as its current version, and the version each marker supersedes moves
+ * to the versionedKeyTable. While versioning is suspended the marker is the
+ * key's null version, so it replaces whatever held that slot: that record is
+ * dropped and its blocks are queued for reclamation. Entries naming a version
+ * are the exception: each permanently deletes that version, as
+ * {@link OMKeyVersionDeleteResponse} does for a single delete.
+ *
+ * The single-key counterpart is {@link OMKeyDeleteMarkerResponse}; both
+ * write out what {@code OMKeyRequest.insertDeleteMarker} produced.
+ */
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, VERSIONED_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE,
+ LIFECYCLE_SCAN_STATE_TABLE})
+public class OMKeysDeleteMarkerResponse extends OmKeyResponse {
+
+ private List insertions;
+ private List versionDeletes;
+ private OmBucketInfo omBucketInfo;
+ private OmLifecycleScanState scanState;
+
+ public OMKeysDeleteMarkerResponse(@Nonnull OMResponse omResponse,
+ @Nonnull List insertions,
+ @Nonnull List versionDeletes,
+ @Nonnull OmBucketInfo omBucketInfo, OmLifecycleScanState scanState) {
+ super(omResponse, BucketLayout.OBJECT_STORE);
+ this.insertions = insertions;
+ this.versionDeletes = versionDeletes;
+ this.omBucketInfo = omBucketInfo;
+ this.scanState = scanState;
+ }
+
+ /**
+ * For when the request is not successful.
+ * For a successful request, the other constructor should be used.
+ */
+ public OMKeysDeleteMarkerResponse(@Nonnull OMResponse omResponse,
+ @Nonnull BucketLayout bucketLayout) {
+ super(omResponse, bucketLayout);
+ checkStatusNotOK();
+ }
+
+ @Override
+ public void checkAndUpdateDB(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ if (getOMResponse().getStatus() == OK
+ || getOMResponse().getStatus() == PARTIAL_DELETE) {
+ addToDBBatch(omMetadataManager, batchOperation);
+ }
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+
+ // Everything the batch sends to the deletedTable, put once per key: the
+ // null version a suspended marker replaced and the versions deleted by id
+ // of the same key are named by the same transaction.
+ Map deletedBlocks = new HashMap<>();
+ for (DeleteMarkerInsertion inserted : insertions) {
+ // The version the marker supersedes and the marker itself go in one
+ // batch, so a reader never sees the key without either.
+ if (inserted.getDemotedVersion() != null) {
+ omMetadataManager.getVersionedKeyTable().putWithBatch(batchOperation,
+ inserted.getDemotedVersionKey(), inserted.getDemotedVersion());
+ }
+ omMetadataManager.getKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, inserted.getObjectKey(), inserted.getDeleteMarker());
+
+ // The null version a suspended marker replaced, if the key had one.
+ if (inserted.getReplacedNullVersionKey() != null) {
+ omMetadataManager.getVersionedKeyTable().deleteWithBatch(
+ batchOperation, inserted.getReplacedNullVersionKey());
+ }
+ if (inserted.getKeysToDelete() != null) {
+ deletedBlocks.putAll(inserted.getKeysToDelete());
+ }
+ }
+
+ // After the markers and in request order, as the request applied them to
+ // the table cache.
+ for (OMKeyVersionDeleteResponse deleted : versionDeletes) {
+ deleted.addToDBBatch(omMetadataManager, batchOperation, deletedBlocks);
+ }
+ for (Map.Entry entry : deletedBlocks.entrySet()) {
+ omMetadataManager.getDeletedTable().putWithBatch(batchOperation, entry.getKey(), entry.getValue());
+ }
+
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
+ omBucketInfo.getBucketName()), omBucketInfo);
+
+ if (scanState != null) {
+ omMetadataManager.getLifecycleScanStateTable().putWithBatch(
+ batchOperation, scanState.getBucketKey(), scanState);
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMObjectVersionsReclaimResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMObjectVersionsReclaimResponse.java
new file mode 100644
index 000000000000..cfcef9f633d6
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMObjectVersionsReclaimResponse.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
+
+import com.google.common.annotations.VisibleForTesting;
+import jakarta.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Response for the reclamation the lifecycle scan selects. Expired noncurrent
+ * versions leave the versionedKeyTable and their blocks go to the deletedTable,
+ * which is the single path through which version blocks are reclaimed:
+ * KeyDeletingService decides from there whether a snapshot still needs them.
+ * Expired delete markers leave the keyTable, and since a marker holds no
+ * blocks, nothing goes to the deletedTable for one.
+ */
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, VERSIONED_KEY_TABLE,
+ DELETED_TABLE, BUCKET_TABLE})
+public class OMObjectVersionsReclaimResponse extends OmKeyResponse {
+
+ private final List reclaimedVersionKeys;
+ private final List reclaimedMarkerKeys;
+ private final Map keysToDelete;
+ private final List updatedBuckets;
+
+ public OMObjectVersionsReclaimResponse(@Nonnull OMResponse omResponse,
+ @Nonnull List reclaimedVersionKeys,
+ @Nonnull List reclaimedMarkerKeys,
+ @Nonnull Map keysToDelete,
+ @Nonnull List updatedBuckets) {
+ super(omResponse, BucketLayout.OBJECT_STORE);
+ this.reclaimedVersionKeys = reclaimedVersionKeys;
+ this.reclaimedMarkerKeys = reclaimedMarkerKeys;
+ this.keysToDelete = keysToDelete;
+ this.updatedBuckets = updatedBuckets;
+ }
+
+ /**
+ * For when the request is not successful.
+ * For a successful request, the other constructor should be used.
+ */
+ public OMObjectVersionsReclaimResponse(@Nonnull OMResponse omResponse) {
+ super(omResponse, BucketLayout.OBJECT_STORE);
+ this.reclaimedVersionKeys = Collections.emptyList();
+ this.reclaimedMarkerKeys = Collections.emptyList();
+ this.keysToDelete = Collections.emptyMap();
+ this.updatedBuckets = Collections.emptyList();
+ checkStatusNotOK();
+ }
+
+ @VisibleForTesting
+ public Map getKeysToDelete() {
+ return keysToDelete;
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ for (String versionKey : reclaimedVersionKeys) {
+ omMetadataManager.getVersionedKeyTable()
+ .deleteWithBatch(batchOperation, versionKey);
+ }
+
+ // An expired marker leaves the keyTable and the key disappears with it.
+ // It holds no blocks, so nothing goes to the deletedTable for it.
+ for (String markerKey : reclaimedMarkerKeys) {
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .deleteWithBatch(batchOperation, markerKey);
+ }
+
+ for (Map.Entry entry : keysToDelete.entrySet()) {
+ omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
+ entry.getKey(), entry.getValue());
+ }
+
+ for (OmBucketInfo omBucketInfo : updatedBuckets) {
+ omMetadataManager.getBucketTable().putWithBatch(batchOperation,
+ omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
+ omBucketInfo.getBucketName()), omBucketInfo);
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
index a1dfe4ed317d..b9da28f2bbb4 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_INFO_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_PARTS_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
@@ -48,7 +49,8 @@
* 3) Delete unused parts.
*/
@CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, KEY_TABLE, DELETED_TABLE,
- MULTIPART_INFO_TABLE, MULTIPART_PARTS_TABLE, BUCKET_TABLE})
+ MULTIPART_INFO_TABLE, MULTIPART_PARTS_TABLE, BUCKET_TABLE,
+ VERSIONED_KEY_TABLE})
public class S3MultipartUploadCompleteResponse extends OmKeyResponse {
private String multipartKey;
private String multipartOpenKey;
@@ -57,6 +59,9 @@ public class S3MultipartUploadCompleteResponse extends OmKeyResponse {
private List multipartPartKeysToDelete;
private OmBucketInfo omBucketInfo;
private long bucketId;
+ private String versionedKeyName;
+ private OmKeyInfo versionedKeyInfo;
+ private String replacedNullVersionKey;
@SuppressWarnings("parameternumber")
public S3MultipartUploadCompleteResponse(
@@ -128,6 +133,19 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
}
}
+ /**
+ * The version this upload superseded, to be kept in the versionedKeyTable as
+ * a noncurrent version, and the null version it replaced, if any. Both null
+ * for buckets that have never been versioned.
+ */
+ public S3MultipartUploadCompleteResponse withVersionedKey(
+ String dbVersionedKey, OmKeyInfo keyInfo, String replacedNullVersion) {
+ this.versionedKeyName = dbVersionedKey;
+ this.versionedKeyInfo = keyInfo;
+ this.replacedNullVersionKey = replacedNullVersion;
+ return this;
+ }
+
protected String addToKeyTable(OMMetadataManager omMetadataManager,
BatchOperation batchOperation) throws IOException {
@@ -135,6 +153,15 @@ protected String addToKeyTable(OMMetadataManager omMetadataManager,
omKeyInfo.getBucketName(), omKeyInfo.getKeyName());
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation, ozoneKey, omKeyInfo);
+
+ if (versionedKeyInfo != null) {
+ omMetadataManager.getVersionedKeyTable()
+ .putWithBatch(batchOperation, versionedKeyName, versionedKeyInfo);
+ }
+ if (replacedNullVersionKey != null) {
+ omMetadataManager.getVersionedKeyTable()
+ .deleteWithBatch(batchOperation, replacedNullVersionKey);
+ }
return ozoneKey;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index c36097fd4d3c..182e24a37889 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -85,6 +85,7 @@
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLCExpiration;
import org.apache.hadoop.ozone.om.helpers.OmLCFilter;
import org.apache.hadoop.ozone.om.helpers.OmLCRule;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
@@ -103,6 +104,9 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionRecord;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionsBucket;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ReclaimObjectVersionsRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
@@ -234,6 +238,15 @@ public BackgroundTaskQueue getTasks() {
return queue;
}
+ /**
+ * Whether the service is switched on at all, which is a matter of
+ * configuration and fixed for the life of the OM. Distinct from being
+ * suspended, which an admin can undo without a restart.
+ */
+ public boolean isEnabled() {
+ return isServiceEnabled.get();
+ }
+
private boolean shouldRun() {
if (getOzoneManager() == null) {
// OzoneManager can be null for testing
@@ -296,6 +309,10 @@ public final class LifecycleActionTask implements BackgroundTask {
private final OmLifecycleConfiguration policy;
private long taskStartTime;
private long numKeyIterated = 0;
+ private long numDeleteMarkersInserted = 0;
+ /** keyTable dbKeys of markers an ExpiredObjectDeleteMarker rule removes. */
+ private final List
+ expiredMarkerKeys = new ArrayList<>();
private long numDirIterated = 0;
private long numDirDeleted = 0;
private long numKeyDeleted = 0;
@@ -306,6 +323,8 @@ public final class LifecycleActionTask implements BackgroundTask {
private long numMultipartUploadIterated = 0;
private long numMultipartUploadAborted = 0;
private String lastScannedKey;
+ // Where the versionedKeyTable scan left off, always at a key boundary.
+ private String lastScannedVersionKey;
private String lastScannedDir;
private String lastScannedDirKey;
@@ -396,6 +415,10 @@ public BackgroundTaskResult call() {
.filter(r -> r.getAbortIncompleteMultipartUpload() != null)
.collect(Collectors.toList());
+ List noncurrentRules = ruleList.stream()
+ .filter(r -> r.getNoncurrentVersionExpiration() != null)
+ .collect(Collectors.toList());
+
if (!expirationRules.isEmpty()) {
LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize);
LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize);
@@ -436,6 +459,8 @@ public BackgroundTaskResult call() {
evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList, scanStateBuilder);
}
+ submitExpiredMarkers(bucket);
+
if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) {
LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey);
sendSaveScanStateRequest(scanStateBuilder, true);
@@ -446,8 +471,8 @@ public BackgroundTaskResult call() {
// If trash is enabled, move files to trash, instead of send delete requests.
// OBS bucket doesn't support trash.
if (bucket.getBucketLayout() == OBJECT_STORE) {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
- bucket.getOwner(), expiredKeyList, false, scanStateBuilder, true);
+ sendDeleteKeysRequestAndClearList(bucket, expiredKeyList, false,
+ scanStateBuilder, true);
} else {
// handle keys first, then directories
handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, true);
@@ -456,6 +481,8 @@ public BackgroundTaskResult call() {
}
}
+ scanNoncurrentVersionsIfAny(bucket, noncurrentRules, scanStateBuilder);
+
if (!mpuRules.isEmpty()) {
processMultipartUploads(bucket, mpuRules);
}
@@ -986,8 +1013,8 @@ private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList exp
boolean saved = false;
if (expiredKeyList != null && !expiredKeyList.isEmpty()) {
if (bucket.getBucketLayout() == OBJECT_STORE) {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
- bucket.getOwner(), expiredKeyList, false, scanStateBuilder, false);
+ sendDeleteKeysRequestAndClearList(bucket, expiredKeyList, false,
+ scanStateBuilder, false);
} else {
handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, false);
}
@@ -1006,6 +1033,159 @@ private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList exp
lastStateSaveKeyCount = numKeyIterated;
}
+ /**
+ * Noncurrent versions live in a table of their own, so they need a scan of
+ * their own. Only a bucket that has been versioned can have any, and
+ * versioning is an OBJECT_STORE feature.
+ */
+ private void scanNoncurrentVersionsIfAny(OmBucketInfo bucket,
+ List noncurrentRules,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ if (!noncurrentRules.isEmpty()
+ && bucket.getBucketLayout() == OBJECT_STORE
+ && bucket.hasEverBeenVersioned()) {
+ evaluateNoncurrentVersions(bucket, noncurrentRules, scanStateBuilder);
+ }
+ }
+
+ /**
+ * Reclaims the noncurrent versions of a versioned bucket's keys, one batch
+ * per pass. Which versions go is {@link NoncurrentVersionSelector}'s
+ * decision; this only feeds it, submits what it picked, and records where
+ * it stopped.
+ */
+ private void evaluateNoncurrentVersions(OmBucketInfo bucket,
+ List ruleList, OmLifecycleScanState.Builder scanStateBuilder) {
+ final NoncurrentVersionSelector selector =
+ new NoncurrentVersionSelector(omMetadataManager);
+ String resumeFrom = scanStateBuilder == null ? null
+ : scanStateBuilder.getLastScannedVersionKey();
+
+ try {
+ while (true) {
+ if (!shouldRun()) {
+ LOG.info("KeyLifecycleService is suspended or disabled. Stopping "
+ + "the noncurrent version scan for bucket {}.",
+ bucket.getBucketName());
+ return;
+ }
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(bucket, ruleList, resumeFrom, listMaxSize);
+ numKeyIterated += selection.getVersionsScanned();
+ lastScannedVersionKey = selection.getResumeFrom();
+ submitExpiredVersions(bucket, selection.getExpiredVersions());
+
+ // Saving state costs a Ratis write, so it is throttled the same way
+ // the keyTable scan throttles it. Resuming from an older boundary
+ // only re-selects versions the reclaim request then skips as gone.
+ if (selection.isFinished() || shouldSaveState()) {
+ saveVersionScanState(scanStateBuilder, selection.isFinished());
+ }
+
+ if (selection.isFinished()) {
+ return;
+ }
+ resumeFrom = selection.getResumeFrom();
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to iterate the noncurrent versions of bucket {}/{}",
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ }
+ }
+
+ /**
+ * Submits the versions one pass selected, if any.
+ */
+ private void submitExpiredVersions(OmBucketInfo bucket,
+ List expiredVersions) {
+ submitForReclaim(bucket, expiredVersions, Collections.emptyList());
+ }
+
+ /**
+ * Submits the expired delete markers the scan selected, if any. They go
+ * through the same request as an expired version rather than through
+ * DeleteKeys, which on a versioned bucket would answer a delete by
+ * inserting another marker. Only a versioned bucket has markers, and
+ * versioning is an OBJECT_STORE feature, so the FSO walk never fills this.
+ */
+ private void submitExpiredMarkers(OmBucketInfo bucket) {
+ submitForReclaim(bucket, Collections.emptyList(), expiredMarkerKeys);
+ }
+
+ /** Each record with the updateID it carried when the scan read it. */
+ private List toProto(
+ List records) {
+ List proto = new ArrayList<>(records.size());
+ for (NoncurrentVersionSelector.ExpiredRecord record : records) {
+ proto.add(ObjectVersionRecord.newBuilder()
+ .setDbKey(record.getDbKey())
+ .setUpdateId(record.getUpdateId())
+ .build());
+ }
+ return proto;
+ }
+
+ private void submitForReclaim(OmBucketInfo bucket,
+ List expiredVersions,
+ List expiredMarkers) {
+ final int submitted = expiredVersions.size() + expiredMarkers.size();
+ if (submitted > 0) {
+ ObjectVersionsBucket versionsBucket = ObjectVersionsBucket.newBuilder()
+ .setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getBucketName())
+ .addAllVersions(toProto(expiredVersions))
+ .addAllMarkers(toProto(expiredMarkers))
+ .build();
+
+ OMRequest omRequestRaw = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.ReclaimObjectVersions)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setReclaimObjectVersionsRequest(
+ ReclaimObjectVersionsRequest.newBuilder()
+ .addVersionsPerBucket(versionsBucket))
+ .build();
+
+ try {
+ final OMClientRequest omClientRequest =
+ OzoneManagerRatisUtils.createClientRequest(omRequestRaw, getOzoneManager());
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(bucket.getOwner());
+ ugi.doAs((PrivilegedExceptionAction) () -> {
+ OMRequest omRequest = omClientRequest.preExecute(getOzoneManager());
+ return OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ });
+ getOzoneManager().getDeletionMetrics()
+ .incrNumObjectVersionsSentForReclaim(submitted);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted while reclaiming {} records of bucket {}/{}",
+ submitted, bucket.getVolumeName(), bucket.getBucketName(), e);
+ } catch (Exception e) {
+ LOG.warn("Failed to reclaim {} records of bucket {}/{}",
+ submitted, bucket.getVolumeName(), bucket.getBucketName(), e);
+ }
+ expiredVersions.clear();
+ expiredMarkers.clear();
+ }
+ }
+
+ /**
+ * Records the key boundary the scan reached, so an interrupted scan
+ * resumes there rather than from the beginning of the bucket.
+ */
+ private void saveVersionScanState(
+ OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (scanStateBuilder != null) {
+ scanStateBuilder.setLastScannedVersionKey(lastScannedVersionKey);
+ }
+ sendSaveScanStateRequest(scanStateBuilder, scanFinished);
+ lastStateSaveTime = Time.monotonicNow();
+ lastStateSaveKeyCount = numKeyIterated;
+ }
+
private void evaluateBucket(OmBucketInfo bucketInfo,
Table keyTable, List ruleList, LimitedExpiredObjectList expiredKeyList,
OmLifecycleScanState.Builder scanStateBuilder) {
@@ -1044,12 +1224,71 @@ private void evaluateBucket(OmBucketInfo bucketInfo,
}
}
+ /**
+ * Selects a delete marker for removal if an ExpiredObjectDeleteMarker rule
+ * covers it and nothing is left under it. "Expired" is not about the
+ * marker's own age: a marker is expired exactly when it is the key's only
+ * remaining version, because then it hides nothing and no versionId can
+ * address it, so nothing else would ever remove it. While a noncurrent
+ * version survives, removing the marker would promote that version and
+ * resurrect an object the user deleted.
+ */
+ private void collectIfExpiredMarker(OmBucketInfo bucketInfo,
+ OmKeyInfo marker, List ruleList) {
+ for (OmLCRule rule : ruleList) {
+ OmLCExpiration expiration = rule.getExpiration();
+ if (expiration == null || !expiration.isExpiredObjectDeleteMarker()
+ || !rule.matchesScope(marker)) {
+ continue;
+ }
+ try {
+ if (hasNoncurrentVersion(bucketInfo, marker)) {
+ return;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to check the versions left under the delete marker "
+ + "of {}/{}/{}", bucketInfo.getVolumeName(),
+ bucketInfo.getBucketName(), marker.getKeyName(), e);
+ return;
+ }
+ expiredMarkerKeys.add(new NoncurrentVersionSelector.ExpiredRecord(
+ omMetadataManager.getOzoneKey(bucketInfo.getVolumeName(),
+ bucketInfo.getBucketName(), marker.getKeyName()),
+ marker.getUpdateID()));
+ if (expiredMarkerKeys.size() >= listMaxSize) {
+ submitExpiredMarkers(bucketInfo);
+ }
+ return;
+ }
+ }
+
+ /** Whether the key has any version left in the versionedKeyTable. */
+ private boolean hasNoncurrentVersion(OmBucketInfo bucketInfo,
+ OmKeyInfo marker) throws IOException {
+ try (Table.KeyValueIterator versions =
+ omMetadataManager.getVersionedKeyTable().iterator(
+ omMetadataManager.getVersionedOzoneKeyPrefix(
+ bucketInfo.getVolumeName(), bucketInfo.getBucketName(),
+ marker.getKeyName()))) {
+ return versions.hasNext();
+ }
+ }
+
private void processKey(OmBucketInfo bucketInfo, OmKeyInfo key, List ruleList,
LimitedExpiredObjectList expiredKeyList, OmLifecycleScanState.Builder scanStateBuilder) {
if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY &&
key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) {
return;
}
+ // Expiring a key whose current version is already a delete marker would
+ // put a marker on top of a marker, and the new one would age into the
+ // rule again, so the chain would grow for as long as the rule exists.
+ // The object is already gone from an unversioned read; removing the
+ // marker that hides it is ExpiredObjectDeleteMarker's decision.
+ if (key.isDeleteMarker()) {
+ collectIfExpiredMarker(bucketInfo, key, ruleList);
+ return;
+ }
for (OmLCRule rule : ruleList) {
if (rule.match(key)) {
// mark key as expired, check next key
@@ -1332,10 +1571,12 @@ private void onFailure(String bucketName) {
metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
long timeSpent = Time.monotonicNow() - taskStartTime;
LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
- "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "deleted {} keys with {} bytes, and {} dirs, inserted {} delete markers, " +
+ "renamed {} keys with {} bytes, and {} dirs to trash, " +
"aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
- numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ numDeleteMarkersInserted, numKeyRenamed, sizeKeyRenamed, numDirRenamed,
+ numMultipartUploadAborted);
}
private void onSuccess(String bucketName) {
@@ -1348,10 +1589,12 @@ private void onSuccess(String bucketName) {
metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
metrics.incNumMultipartUploadAborted(numMultipartUploadAborted);
LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
- "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "deleted {} keys with {} bytes, and {} dirs, inserted {} delete markers, " +
+ "renamed {} keys with {} bytes, and {} dirs to trash, " +
"aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
- numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ numDeleteMarkersInserted, numKeyRenamed, sizeKeyRenamed, numDirRenamed,
+ numMultipartUploadAborted);
}
private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList,
@@ -1360,8 +1603,8 @@ private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectLis
moveToTrash(bucket, keysList, dir);
sendSaveScanStateRequest(scanStateBuilder, scanFinished);
} else {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
- bucket.getOwner(), keysList, dir, scanStateBuilder, scanFinished);
+ sendDeleteKeysRequestAndClearList(bucket, keysList, dir, scanStateBuilder,
+ scanFinished);
}
}
@@ -1401,9 +1644,19 @@ private void sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuil
}
}
- private void sendDeleteKeysRequestAndClearList(String volume, String bucket, String bucketOwner,
+ /**
+ * On a bucket that has ever been versioned this request inserts a delete
+ * marker per key instead of deleting: the object stops being visible, but
+ * its current version is demoted rather than removed, so nothing is
+ * reclaimed and no space is freed.
+ */
+ private void sendDeleteKeysRequestAndClearList(OmBucketInfo bucketInfo,
LimitedExpiredObjectList keysList, boolean dir,
OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ final String volume = bucketInfo.getVolumeName();
+ final String bucket = bucketInfo.getBucketName();
+ final String bucketOwner = bucketInfo.getOwner();
+ final boolean insertsMarkers = bucketInfo.hasEverBeenVersioned();
try {
if (getInjector(1) != null) {
try {
@@ -1517,6 +1770,9 @@ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
if (dir) {
numDirDeleted += deletedCount;
metrics.incrNumDirDeleted(deletedCount);
+ } else if (insertsMarkers) {
+ numDeleteMarkersInserted += deletedCount;
+ metrics.incrNumDeleteMarkersInserted(deletedCount);
} else {
numKeyDeleted += deletedCount;
sizeKeyDeleted += deletedSize;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java
index b4ffe055396a..6e8e06c98fad 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java
@@ -60,6 +60,8 @@ public final class KeyLifecycleServiceMetrics {
private MutableGaugeLong numDirRenamed;
@Metric("Total size of keys deleted")
private MutableGaugeLong sizeKeyDeleted;
+ @Metric("Total delete markers inserted in place of deleting an expired object")
+ private MutableGaugeLong numDeleteMarkersInserted;
@Metric("Total size of keys renamed")
private MutableGaugeLong sizeKeyRenamed;
@Metric("Number of multipart uploads iterated")
@@ -126,6 +128,19 @@ public void incrSizeKeyDeleted(long size) {
sizeKeyDeleted.incr(size);
}
+ /**
+ * Expiring an object on a bucket that has ever been versioned inserts a
+ * delete marker; it deletes nothing and frees no space, so it is counted
+ * here rather than as a deletion.
+ */
+ public void incrNumDeleteMarkersInserted(long markerCount) {
+ numDeleteMarkersInserted.incr(markerCount);
+ }
+
+ public MutableGaugeLong getNumDeleteMarkersInserted() {
+ return numDeleteMarkersInserted;
+ }
+
public void incrSizeKeyRenamed(long size) {
sizeKeyRenamed.incr(size);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/NoncurrentVersionSelector.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/NoncurrentVersionSelector.java
new file mode 100644
index 000000000000..f8b0bf6cf7ea
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/NoncurrentVersionSelector.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.service;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_VERSIONED_KEY_SEPARATOR;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLCNoncurrentVersionExpiration;
+import org.apache.hadoop.ozone.om.helpers.OmLCRule;
+
+/**
+ * Decides which of a bucket's noncurrent versions a set of
+ * NoncurrentVersionExpiration rules expires.
+ *
+ * The versionedKeyTable holds a key's versions adjacent and newest first,
+ * so one pass gives the position a NewerNoncurrentVersions rule counts:
+ * position N is the Nth newest noncurrent version. The moment a version
+ * stopped being current is on the record itself, stamped when it was moved
+ * out of the keyTable.
+ *
+ *
Separate from the scan that drives it so that what gets selected can be
+ * checked without running a lifecycle service.
+ */
+public class NoncurrentVersionSelector {
+
+ private final OMMetadataManager metadataManager;
+
+ public NoncurrentVersionSelector(OMMetadataManager metadataManager) {
+ this.metadataManager = metadataManager;
+ }
+
+ /**
+ * A record the rules expired, named by its dbKey and by the write that
+ * produced it.
+ *
+ *
The dbKey alone does not identify it. Scan and reclaim are separated by
+ * replication, and a dbKey names a position rather than a record: the key
+ * can be rewritten in between, leaving a different record where the scan
+ * looked. The updateId is carried so the reclaim can tell the two apart.
+ */
+ public static final class ExpiredRecord {
+ private final String dbKey;
+ private final long updateId;
+
+ public ExpiredRecord(String dbKey, long updateId) {
+ this.dbKey = dbKey;
+ this.updateId = updateId;
+ }
+
+ public String getDbKey() {
+ return dbKey;
+ }
+
+ public long getUpdateId() {
+ return updateId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ExpiredRecord other = (ExpiredRecord) o;
+ return updateId == other.updateId && dbKey.equals(other.dbKey);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dbKey, updateId);
+ }
+
+ @Override
+ public String toString() {
+ return dbKey + "@" + updateId;
+ }
+ }
+
+ /**
+ * What one pass selected, and where it stopped.
+ */
+ public static final class Selection {
+ private final List expiredVersions;
+ private final String resumeFrom;
+ private final long versionsScanned;
+
+ private Selection(List expiredVersions, String resumeFrom,
+ long versionsScanned) {
+ this.expiredVersions = expiredVersions;
+ this.resumeFrom = resumeFrom;
+ this.versionsScanned = versionsScanned;
+ }
+
+ /**
+ * @return how many versions the pass walked, expired or not. The scan
+ * reports this as the work it did: a pass over versions that all
+ * survive is as much work as one that expires every version it sees.
+ */
+ public long getVersionsScanned() {
+ return versionsScanned;
+ }
+
+ /** The versions the rules expire, in the order the pass met them. */
+ public List getExpiredVersions() {
+ return expiredVersions;
+ }
+
+ /**
+ * @return the dbKey to resume the next pass from, always the first version
+ * of a key, or null when the bucket was walked to the end. Never points
+ * into the middle of a key: resuming there would restart the per-key
+ * count and keep more versions than the rule allows.
+ */
+ public String getResumeFrom() {
+ return resumeFrom;
+ }
+
+ public boolean isFinished() {
+ return resumeFrom == null;
+ }
+ }
+
+ /**
+ * Walks the bucket's noncurrent versions and selects the expired ones.
+ *
+ * @param bucket the bucket to walk
+ * @param rules the rules carrying a NoncurrentVersionExpiration action
+ * @param resumeFrom where a previous pass stopped, or null to start over
+ * @param limit how many versions to walk before stopping at the next key
+ * boundary; the pass can exceed it by the remainder of the key being
+ * walked, so that a key is never left half-processed. It bounds the
+ * versions read rather than the ones selected, so a bucket whose
+ * versions all survive still yields the scan between passes instead of
+ * being walked to the end in one.
+ */
+ public Selection select(OmBucketInfo bucket, List rules,
+ String resumeFrom, int limit) throws IOException {
+
+ // Trailing separator included: without it the prefix of a bucket is also
+ // a prefix of every bucket whose name extends it, so scanning one would
+ // expire another's versions under this bucket's rules.
+ final String bucketPrefix = metadataManager.getBucketKey(
+ bucket.getVolumeName(), bucket.getBucketName()) + OM_KEY_PREFIX;
+ final List expired = new ArrayList<>();
+
+ String currentKeyName = null;
+ int seenInKey = 0;
+ long scanned = 0;
+
+ try (Table.KeyValueIterator versions =
+ metadataManager.getVersionedKeyTable().iterator(bucketPrefix)) {
+ if (resumeFrom != null) {
+ versions.seek(resumeFrom);
+ }
+
+ while (versions.hasNext()) {
+ Table.KeyValue entry = versions.next();
+ final String dbKey = entry.getKey();
+ // The versionId suffix is fixed-width lower-case hex, so the last
+ // separator is the one that starts it. Searching forward would cut
+ // the name short on a key that contains the separator itself.
+ final int separator = dbKey.lastIndexOf(OM_VERSIONED_KEY_SEPARATOR);
+ if (separator < 0) {
+ continue;
+ }
+ final String keyName =
+ dbKey.substring(bucketPrefix.length(), separator);
+
+ if (!keyName.equals(currentKeyName)) {
+ // A key boundary, the only place a pass may stop.
+ if (scanned >= limit) {
+ return new Selection(expired, dbKey, scanned);
+ }
+ currentKeyName = keyName;
+ seenInKey = 0;
+ }
+
+ final OmKeyInfo version = entry.getValue();
+ scanned++;
+ if (isExpired(rules, version, ++seenInKey)) {
+ expired.add(new ExpiredRecord(dbKey, version.getUpdateID()));
+ }
+ }
+ }
+
+ // Walked to the end, so the next pass starts over rather than resuming.
+ return new Selection(expired, null, scanned);
+ }
+
+ /**
+ * @return whether a rule that matches this version expires it, either
+ * because enough newer noncurrent versions of the key survive it or
+ * because it has been noncurrent long enough.
+ */
+ private boolean isExpired(List rules, OmKeyInfo version,
+ int position) {
+ final long becameNoncurrentAt = version.getNoncurrentTime();
+ for (OmLCRule rule : rules) {
+ // Scope only: whether the version is expired is this action's own
+ // decision, not the Expiration action's modification-time check.
+ if (!rule.matchesScope(version)) {
+ continue;
+ }
+ final OmLCNoncurrentVersionExpiration expiration =
+ rule.getNoncurrentVersionExpiration();
+ final Integer keep = expiration.getNewerNoncurrentVersions();
+ if (keep != null && position > keep) {
+ return true;
+ }
+ // Every record in the versionedKeyTable is stamped as it is moved
+ // there, so an unstamped one means that invariant is broken. Expiring
+ // on a broken invariant would destroy data, so it is left alone.
+ if (becameNoncurrentAt > 0
+ && expiration.isExpired(becameNoncurrentAt)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 189bb3f3e1d5..c03def8b931f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@ -46,7 +46,10 @@ public enum OMLayoutFeature implements LayoutFeature {
DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"),
SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"),
S3_LIFECYCLE_SUPPORT(10, "S3 bucket lifecycle configuration support"),
- MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for parts and key");
+ MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for parts and key"),
+
+ OBJECT_VERSIONING(12, "S3-compatible object versioning: bucket versioning"
+ + " state machine and the versionedKeyTable for noncurrent versions");
/////////////////////////////// /////////////////////////////
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index b585fa234c8b..463de53e4e9f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -655,6 +655,7 @@ private LookupKeyResponse lookupKey(LookupKeyRequest request,
LookupKeyResponse.Builder resp =
LookupKeyResponse.newBuilder();
KeyArgs keyArgs = request.getKeyArgs();
+ OmKeyArgs.validateAddressedVersion(keyArgs);
OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
.setVolumeName(keyArgs.getVolumeName())
.setBucketName(keyArgs.getBucketName())
@@ -662,6 +663,8 @@ private LookupKeyResponse lookupKey(LookupKeyRequest request,
.setLatestVersionLocation(keyArgs.getLatestVersionLocation())
.setSortDatanodesInPipeline(keyArgs.getSortDatanodes())
.setHeadOp(keyArgs.getHeadOp())
+ .setVersionId(keyArgs.hasVersionId() ? keyArgs.getVersionId() : null)
+ .setNullVersion(keyArgs.getNullVersion())
.build();
OmKeyInfo keyInfo = impl.lookupKey(omKeyArgs);
@@ -673,6 +676,7 @@ private LookupKeyResponse lookupKey(LookupKeyRequest request,
private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request,
int clientVersion) throws IOException {
KeyArgs keyArgs = request.getKeyArgs();
+ OmKeyArgs.validateAddressedVersion(keyArgs);
OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
.setVolumeName(keyArgs.getVolumeName())
.setBucketName(keyArgs.getBucketName())
@@ -683,6 +687,8 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request,
.setForceUpdateContainerCacheFromSCM(
keyArgs.getForceUpdateContainerCacheFromSCM())
.setMultipartUploadPartNumber(keyArgs.getMultipartNumber())
+ .setVersionId(keyArgs.hasVersionId() ? keyArgs.getVersionId() : null)
+ .setNullVersion(keyArgs.getNullVersion())
.build();
KeyInfoWithVolumeContext keyInfo = impl.getKeyInfo(omKeyArgs,
request.getAssumeS3Context());
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
index e7df8077cc37..3771d023891e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -23,6 +23,8 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.anySet;
import static org.mockito.Mockito.mock;
@@ -64,6 +66,7 @@
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -93,6 +96,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
/**
* Unit test key manager.
@@ -581,10 +586,355 @@ public void testGetKeyInfo() throws IOException {
.getContainerWithPipelineBatch(containerIDs);
}
+ private OmKeyInfo versionedKeyInfo(String volume, String bucket, String key,
+ long versionId, boolean nullVersion, boolean deleteMarker) {
+ return new OmKeyInfo.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .setOmKeyLocationInfos(Collections.emptyList())
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setDataSize(0)
+ .setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE))
+ .setVersionId(versionId)
+ .setNullVersion(nullVersion)
+ .setDeleteMarker(deleteMarker)
+ .build();
+ }
+
+ /**
+ * Enabling versioning does not rewrite the objects a bucket already holds:
+ * they keep no versionId at all, and S3 reports their version as "null".
+ */
+ @Test
+ public void testLookupPreVersioningKeyAsNullVersion() throws Exception {
+ String volume = "vol-legacy";
+ String bucket = "buck-legacy";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // written before versioning was enabled: no versionId, no null flag
+ OmKeyInfo legacy = new OmKeyInfo.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .setOmKeyLocationInfos(Collections.emptyList())
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setDataSize(0)
+ .setReplicationConfig(
+ RatisReplicationConfig.getInstance(ReplicationFactor.ONE))
+ .build();
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key), legacy);
+
+ OmKeyArgs.Builder base = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true);
+
+ // a plain read still returns it
+ OmKeyArgs args = base.build();
+ assertNull(keyManager.lookupKey(args, resolveBucket(args), null)
+ .getVersionId());
+
+ // and version "null" addresses it, without the record having been rewritten
+ args = base.setNullVersion(true).build();
+ OmKeyInfo nullVersion =
+ keyManager.lookupKey(args, resolveBucket(args), null);
+ assertNull(nullVersion.getVersionId());
+ assertTrue(nullVersion.isNullVersionRecord());
+ assertEquals(legacy, metadataManager
+ .getKeyTable(BucketLayout.OBJECT_STORE)
+ .get(metadataManager.getOzoneKey(volume, bucket, key)));
+ }
+
+ @Test
+ public void testLookupKeyByVersionId() throws Exception {
+ String volume = "vol-ver";
+ String bucket = "buck-ver";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // current version 30, one noncurrent version 20, and a null version 10
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 30L, false, false));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 20L),
+ versionedKeyInfo(volume, bucket, key, 20L, false, false));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 10L),
+ versionedKeyInfo(volume, bucket, key, 10L, true, false));
+
+ OmKeyArgs.Builder base = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true);
+
+ // no versionId addresses the current version
+ OmKeyArgs args = base.build();
+ assertEquals(30L, keyManager.lookupKey(args, resolveBucket(args), null)
+ .getVersionId());
+
+ // the current version can also be addressed by its id
+ args = base.setVersionId(30L).build();
+ assertEquals(30L, keyManager.lookupKey(args, resolveBucket(args), null)
+ .getVersionId());
+
+ // a noncurrent version resolves through the versionedKeyTable
+ args = base.setVersionId(20L).build();
+ assertEquals(20L, keyManager.lookupKey(args, resolveBucket(args), null)
+ .getVersionId());
+
+ // the null version slot is found by attribute, not by id
+ args = base.setNullVersion(true).build();
+ OmKeyInfo nullVersion =
+ keyManager.lookupKey(args, resolveBucket(args), null);
+ assertEquals(10L, nullVersion.getVersionId());
+ assertTrue(nullVersion.isNullVersion());
+
+ // an unknown versionId is a plain not-found
+ OmKeyArgs unknown = base.setVersionId(999L).build();
+ OMException ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(unknown, resolveBucket(unknown), null));
+ assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex.getResult());
+ }
+
+ @Test
+ public void testNullVersionIsVisibleInTheCacheWindow() throws Exception {
+ String volume = "vol-ver-cache";
+ String bucket = "buck-ver-cache";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 30L, false, false));
+ // the null version was demoted by a transaction the double buffer has not
+ // flushed yet, so it exists in the cache only
+ String cachedNullVersion =
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 10L);
+ metadataManager.getVersionedKeyTable().addCacheEntry(cachedNullVersion,
+ versionedKeyInfo(volume, bucket, key, 10L, true, false), 1L);
+
+ OmKeyArgs.Builder base = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true);
+
+ OmKeyArgs args = base.setNullVersion(true).build();
+ OmKeyInfo nullVersion =
+ keyManager.lookupKey(args, resolveBucket(args), null);
+ assertEquals(10L, nullVersion.getVersionId());
+ assertTrue(nullVersion.isNullVersion());
+ }
+
+ @Test
+ public void testNullVersionDeletedInTheCacheWindowIsGone() throws Exception {
+ String volume = "vol-ver-tombstone";
+ String bucket = "buck-ver-tombstone";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 30L, false, false));
+ String flushedNullVersion =
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 10L);
+ metadataManager.getVersionedKeyTable().put(flushedNullVersion,
+ versionedKeyInfo(volume, bucket, key, 10L, true, false));
+ // removed by a transaction that is not flushed yet: the DB still has it
+ metadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(flushedNullVersion), CacheValue.get(1L));
+
+ OmKeyArgs args = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true).setNullVersion(true).build();
+ OMException ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(args, resolveBucket(args), null));
+ assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex.getResult());
+ }
+
+ @Test
+ public void testLookupOfDeleteMarkerIsDistinguishable() throws Exception {
+ String volume = "vol-marker";
+ String bucket = "buck-marker";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // current version is a marker, with a readable version behind it
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 40L, false, true));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 20L),
+ versionedKeyInfo(volume, bucket, key, 20L, false, false));
+
+ OmKeyArgs.Builder base = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true);
+
+ // without a versionId a current marker reads as absent
+ OmKeyArgs current = base.build();
+ OMException ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(current, resolveBucket(current), null));
+ assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex.getResult());
+
+ // naming the marker's version is a different condition: S3 answers 405
+ OmKeyArgs marker = base.setVersionId(40L).build();
+ ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(marker, resolveBucket(marker), null));
+ assertEquals(OMException.ResultCodes.KEY_IS_DELETE_MARKER, ex.getResult());
+
+ // the version behind the marker stays readable
+ OmKeyArgs behind = base.setVersionId(20L).build();
+ assertEquals(20L,
+ keyManager.lookupKey(behind, resolveBucket(behind), null).getVersionId());
+
+ // a marker that is no longer current is the same condition: the marker is
+ // addressable wherever it sits, and never has a body to return
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 30L),
+ versionedKeyInfo(volume, bucket, key, 30L, false, true));
+ OmKeyArgs noncurrentMarker = base.setVersionId(30L).build();
+ ex = assertThrows(OMException.class, () -> keyManager.lookupKey(
+ noncurrentMarker, resolveBucket(noncurrentMarker), null));
+ assertEquals(OMException.ResultCodes.KEY_IS_DELETE_MARKER, ex.getResult());
+ }
+
+ /**
+ * A delete while versioning is suspended leaves the marker in the null slot.
+ * It hides the key from plain reads exactly as a versioned marker does, and
+ * the versions behind it stay readable by id.
+ */
+ @Test
+ public void testLookupBehindANullDeleteMarker() throws Exception {
+ String volume = "vol-nullmarker";
+ String bucket = "buck-nullmarker";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // the marker took the null slot, over a version from the enabled era
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 50L, true, true));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 20L),
+ versionedKeyInfo(volume, bucket, key, 20L, false, false));
+
+ OmKeyArgs.Builder base = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true);
+
+ // a plain read steps over the marker and reports the key absent
+ OmKeyArgs plain = base.build();
+ OMException ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(plain, resolveBucket(plain), null));
+ assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex.getResult());
+
+ // addressing the null slot names the marker itself
+ OmKeyArgs nullSlot = base.setNullVersion(true).build();
+ ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(nullSlot, resolveBucket(nullSlot), null));
+ assertEquals(OMException.ResultCodes.KEY_IS_DELETE_MARKER, ex.getResult());
+
+ // and the enabled-era version behind it is still readable
+ OmKeyArgs behind = base.setVersionId(20L).build();
+ assertEquals(20L, keyManager.lookupKey(behind, resolveBucket(behind), null)
+ .getVersionId());
+ }
+
+ @Test
+ public void testCurrentVersionIsResolvedWithoutReadingVersionedKeyTable()
+ throws Exception {
+ String volume = "vol-cur";
+ String bucket = "buck-cur";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // The current version and a decoy stored under the dbKey that version would
+ // occupy in the versionedKeyTable. Resolving to the current record proves
+ // the lookup answers from keyTable and never falls through to the scan.
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 30L, false, false));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 30L),
+ versionedKeyInfo(volume, bucket, key, 30L, false, true));
+
+ OmKeyArgs args = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true).setVersionId(30L).build();
+
+ // the decoy is a delete marker, so reading it would have thrown
+ assertEquals(30L, keyManager.lookupKey(args, resolveBucket(args), null)
+ .getVersionId());
+ }
+
+ @Test
+ public void testNullVersionSlotCanBeTheCurrentVersion() throws Exception {
+ String volume = "vol-null";
+ String bucket = "buck-null";
+ String key = "obj";
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ BucketLayout.OBJECT_STORE);
+
+ // A suspended PUT leaves the null version as the current one, with the
+ // versions accumulated while enabled behind it.
+ metadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volume, bucket, key),
+ versionedKeyInfo(volume, bucket, key, 50L, true, false));
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volume, bucket, key, 20L),
+ versionedKeyInfo(volume, bucket, key, 20L, false, false));
+
+ OmKeyArgs args = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName(key)
+ .setHeadOp(true).setNullVersion(true).build();
+
+ OmKeyInfo nullVersion = keyManager.lookupKey(args, resolveBucket(args), null);
+ assertEquals(50L, nullVersion.getVersionId());
+ assertTrue(nullVersion.isNullVersion());
+ }
+
+ /** Versions live in the OBJECT_STORE layout only; the others cannot hold one. */
+ @ParameterizedTest
+ @EnumSource(value = BucketLayout.class,
+ names = {"FILE_SYSTEM_OPTIMIZED", "LEGACY"})
+ public void testVersionIdRejectedOnNonObjectStoreBucket(BucketLayout layout)
+ throws Exception {
+ String volume = "vol-" + layout.name().toLowerCase();
+ String bucket = "buck-" + layout.name().toLowerCase();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket, metadataManager,
+ layout);
+
+ OmKeyArgs args = new OmKeyArgs.Builder()
+ .setVolumeName(volume).setBucketName(bucket).setKeyName("obj")
+ .setHeadOp(true).setVersionId(1L).build();
+ ResolvedBucket resolved =
+ new ResolvedBucket(volume, bucket, volume, bucket, "", layout);
+
+ OMException ex = assertThrows(OMException.class,
+ () -> keyManager.lookupKey(args, resolved, null));
+ assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION,
+ ex.getResult());
+ }
+
+ /**
+ * Resolves to OBJECT_STORE, the layout the versioning tests create their
+ * buckets with. BucketLayout.DEFAULT is LEGACY, which shares this lookup
+ * path but may not address a version.
+ */
private ResolvedBucket resolveBucket(OmKeyArgs keyArgs) {
return new ResolvedBucket(keyArgs.getVolumeName(), keyArgs.getBucketName(),
keyArgs.getVolumeName(), keyArgs.getBucketName(), "",
- BucketLayout.DEFAULT);
+ BucketLayout.OBJECT_STORE);
}
@Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 1ffcc32f0acb..db288f653c2f 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -48,6 +48,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.TENANT_STATE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.TRANSACTION_INFO_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.USER_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VERSIONED_KEY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VOLUME_TABLE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
@@ -127,6 +128,7 @@ public class TestOmMetadataManager {
VOLUME_TABLE,
BUCKET_TABLE,
KEY_TABLE,
+ VERSIONED_KEY_TABLE,
DELETED_TABLE,
OPEN_KEY_TABLE,
MULTIPART_INFO_TABLE,
@@ -181,6 +183,42 @@ public void testTransactionTable() throws Exception {
assertEquals(250, transactionInfo.getTransactionIndex());
}
+ @Test
+ public void testVersionedOzoneKeyOrdering() {
+ String prefix = omMetadataManager.getVersionedOzoneKeyPrefix("vol", "buck", "key");
+ assertEquals("/vol/buck/key\0", prefix);
+
+ // newer versions (larger versionId) must sort before older ones, and all
+ // versioned keys must sort under the key's prefix
+ String v1 = omMetadataManager.getVersionedOzoneKey("vol", "buck", "key", 1L);
+ String v2 = omMetadataManager.getVersionedOzoneKey("vol", "buck", "key", 42L);
+ String v3 = omMetadataManager.getVersionedOzoneKey("vol", "buck", "key", Long.MAX_VALUE - 1);
+ assertThat(v3).startsWith(prefix).isLessThan(v2);
+ assertThat(v2).startsWith(prefix).isLessThan(v1);
+
+ // fixed-width suffix: identical length regardless of versionId magnitude
+ assertEquals(v1.length(), v3.length());
+ }
+
+ @Test
+ public void testVersionedOzoneKeyIsolatedFromNestedKeys() {
+ // OBJECT_STORE key names contain '/' verbatim, so "key" and "key/001" are two
+ // unrelated keys. Every version of "key" must sort under "key"'s prefix and
+ // ahead of anything belonging to "key/001", otherwise a prefix seek for the
+ // newest noncurrent version of "key" would land on a version of "key/001".
+ String prefix = omMetadataManager.getVersionedOzoneKeyPrefix("vol", "buck", "key");
+ String nestedPrefix = omMetadataManager.getVersionedOzoneKeyPrefix("vol", "buck", "key/001");
+ assertThat(nestedPrefix).doesNotStartWith(prefix);
+
+ String oldest = omMetadataManager.getVersionedOzoneKey("vol", "buck", "key", 1L);
+ String nestedNewest =
+ omMetadataManager.getVersionedOzoneKey("vol", "buck", "key/001", Long.MAX_VALUE - 1);
+ assertThat(oldest).isLessThan(nestedNewest);
+
+ // the nested key's own current entry in keyTable also sorts after all of them
+ assertThat(oldest).isLessThan(omMetadataManager.getOzoneKey("vol", "buck", "key/001"));
+ }
+
@Test
public void testListVolumes() throws Exception {
String ownerName = "owner";
@@ -1581,4 +1619,19 @@ public void testListKeysWithWhitespaceAndNewlinePrefix() throws Exception {
.collect(Collectors.toList());
assertEquals(Collections.singletonList(newlinePrefixed), newlinePrefixMatches);
}
+
+ /**
+ * The SST filter trims a snapshot's versionedKeyTable to its bucket, by the same prefix as the keyTable, which its
+ * keys share.
+ */
+ @Test
+ public void testVersionedKeyTableHasTheBucketKeyPrefix() throws Exception {
+ OMRequestTestUtils.addVolumeAndBucketToDB("vol1", "bucket1", omMetadataManager);
+ String bucketPrefix = omMetadataManager.getBucketKeyPrefix("vol1", "bucket1");
+
+ assertEquals(bucketPrefix,
+ omMetadataManager.getTableBucketPrefix("vol1", "bucket1").getTablePrefix(VERSIONED_KEY_TABLE));
+ assertEquals(bucketPrefix, omMetadataManager.getTableBucketPrefix(VERSIONED_KEY_TABLE, "vol1", "bucket1"));
+ assertTrue(omMetadataManager.getVersionedOzoneKey("vol1", "bucket1", "key", 1L).startsWith(bucketPrefix));
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVersionIdAllocator.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVersionIdAllocator.java
new file mode 100644
index 000000000000..e8cbcbfa6507
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVersionIdAllocator.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.om.helpers.PinnedFirstVersionIdGenerator.FIRST_VERSION_ID;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Deque;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.PinnedFirstVersionIdGenerator;
+import org.apache.hadoop.ozone.om.helpers.UniqueIdVersionIdGenerator;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests {@link VersionIdAllocator}: a proposal is a floor, and the id a version
+ * is applied with always comes after the key's current version, however the
+ * clock behaved.
+ */
+public class TestVersionIdAllocator {
+
+ private static final String VOLUME = "vol1";
+ private static final String BUCKET = "bucket1";
+ private static final String KEY = "key1";
+
+ /** Hands out the readings it was given, standing in for a wall clock. */
+ private static VersionIdGenerator clockReading(long... readings) {
+ final Deque queue = new ArrayDeque<>();
+ Arrays.stream(readings).forEach(queue::add);
+ return () -> queue.size() > 1 ? queue.poll() : queue.peek();
+ }
+
+ private static VersionIdAllocator allocator() {
+ return new VersionIdAllocator(new UniqueIdVersionIdGenerator());
+ }
+
+ private static OmKeyInfo versionWithId(Long versionId) {
+ OmKeyInfo.Builder builder = new OmKeyInfo.Builder()
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setKeyName(KEY)
+ .setReplicationConfig(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
+ if (versionId != null) {
+ builder.setVersionId(versionId);
+ }
+ return builder.build();
+ }
+
+ @Test
+ void theProposalComesFromTheGenerator() {
+ VersionIdAllocator allocator =
+ new VersionIdAllocator(clockReading(1234L, 5678L));
+
+ assertEquals(1234L, allocator.propose());
+ assertEquals(5678L, allocator.propose());
+ }
+
+ @Test
+ void theFirstVersionOfAKeyTakesTheProposedId() {
+ assertEquals(5000L, allocator().allocate(5000L, null));
+ }
+
+ @Test
+ void aLaterVersionTakesTheProposedIdWhenTheClockMovedOn() {
+ assertEquals(5000L, allocator().allocate(5000L, versionWithId(4000L)));
+ }
+
+ /**
+ * Two versions of one key proposed inside the same millisecond can exhaust
+ * the counter that separates them; the second still has to come after the
+ * first.
+ */
+ @Test
+ void aRepeatedProposalStillMovesTheKeyForward() {
+ assertEquals(4001L, allocator().allocate(4000L, versionWithId(4000L)));
+ }
+
+ /**
+ * The case no wall clock handles on its own: a leader change onto a node
+ * whose clock lags, which would otherwise sort the new version before the one
+ * it supersedes.
+ */
+ @Test
+ void aClockThatWentBackwardsDoesNotReorderTheKey() {
+ assertEquals(9001L, allocator().allocate(3000L, versionWithId(9000L)));
+ }
+
+ @Test
+ void everyAllocationComesAfterTheCurrentVersion() {
+ VersionIdAllocator allocator = allocator();
+ long currentId = 1_000_000L;
+
+ for (long proposed : new long[] {2_000_000L, 1L, 1_000_000L, 2_000_001L}) {
+ long allocated = allocator.allocate(proposed, versionWithId(currentId));
+ assertTrue(allocated > currentId,
+ "allocated " + allocated + " does not come after " + currentId);
+ currentId = allocated;
+ }
+ }
+
+ /**
+ * A current version written before versioning was enabled carries no id, and
+ * a key in that state has no other versions to order against.
+ */
+ @Test
+ void aVersionPredatingVersioningDoesNotConstrainTheId() {
+ assertEquals(7000L, allocator().allocate(7000L, versionWithId(null)));
+ }
+
+ /**
+ * A request that reaches apply without a proposal would otherwise take the
+ * reserved unset value as its id, and read back as a version that predates
+ * versioning.
+ */
+ @Test
+ void aVersionCannotBeAppliedWithoutAProposal() {
+ assertThrows(IllegalArgumentException.class,
+ () -> allocator().allocate(VersionIdGenerator.UNSET_VERSION_ID, null));
+ }
+
+ /**
+ * The floor comes from the current version the write path already holds, so
+ * the allocator reads nothing of its own - versionedKeyTable included. It
+ * holds no table to read one from.
+ */
+ @Test
+ void theAllocatorReadsNoTable() {
+ for (Field field : VersionIdAllocator.class.getDeclaredFields()) {
+ if (field.isSynthetic()) {
+ continue;
+ }
+ assertEquals(VersionIdGenerator.class, field.getType(),
+ "the allocator should hold nothing but its generator, but holds "
+ + field.getType().getName() + " " + field.getName());
+ }
+ }
+
+ /**
+ * Whether the key has a current version is settled under the write's lock, so
+ * a generator that numbers the first one specially decides it here.
+ */
+ @Test
+ void thePinnedGeneratorGivesTheFirstVersionTheSentinel() {
+ VersionIdAllocator allocator =
+ new VersionIdAllocator(new PinnedFirstVersionIdGenerator());
+
+ assertEquals(FIRST_VERSION_ID, allocator.allocate(5000L, null));
+ }
+
+ @Test
+ void thePinnedGeneratorLeavesLaterVersionsAlone() {
+ VersionIdAllocator allocator =
+ new VersionIdAllocator(new PinnedFirstVersionIdGenerator());
+
+ assertEquals(5000L,
+ allocator.allocate(5000L, versionWithId(FIRST_VERSION_ID)));
+ }
+
+ @Test
+ void theDefaultGeneratorDoesNotPinTheFirstVersion() {
+ assertEquals(5000L, allocator().allocate(5000L, null));
+ }
+
+ @Test
+ void theAllocatorTakesItsGeneratorFromConfiguration() {
+ assertInstanceOf(UniqueIdVersionIdGenerator.class,
+ new VersionIdAllocator(new OzoneConfiguration()).getGenerator());
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
index 96a60b647d2b..9957d9a92b0a 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
@@ -413,6 +413,29 @@ public void testIgnoreClientACL(boolean ignoreClientACLs) throws Exception {
}
}
+ @Test
+ public void testVersioningStatusRejectedAtCreation() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ addCreateVolumeToTable(volumeName, omMetadataManager);
+
+ OMRequest request = newCreateBucketRequest(
+ newBucketInfoBuilder(bucketName, volumeName)
+ .setVersioningStatus(OzoneManagerProtocolProtos
+ .BucketVersioningStatusProto.VERSIONING_ENABLED)).build();
+
+ OMException omException = assertThrows(OMException.class,
+ () -> new OMBucketCreateRequest(request).preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST,
+ omException.getResult());
+
+ // the same request without a status is accepted
+ OMRequest withoutStatus = newCreateBucketRequest(
+ newBucketInfoBuilder(bucketName, volumeName)).build();
+ assertNotNull(
+ new OMBucketCreateRequest(withoutStatus).preExecute(ozoneManager));
+ }
+
private void acceptBucketCreationHelper(String volumeName, String bucketName)
throws Exception {
OMBucketCreateRequest omBucketCreateRequest =
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
index 2e41d4c8b173..941d2205596e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
@@ -23,26 +23,48 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.UUID;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.service.KeyLifecycleService;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
/**
* Tests OMBucketSetPropertyRequest class which handles OMSetBucketProperty
@@ -422,6 +444,478 @@ public void testValidateAndUpdateCacheWithQuotaNamespaceUsed()
"is less than used namespaceQuota");
}
+ /**
+ * S3 object versioning is only defined for OBJECT_STORE buckets: combining
+ * it with the directory and rename semantics of the other layouts is out of
+ * scope, so the status cannot be set on them at all.
+ */
+ @Test
+ public void testVersioningStatusRejectedOnNonObjectStoreBucket()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED);
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED)).validateAndUpdateCache(
+ ozoneManager, 1);
+
+ assertFalse(response.getOMResponse().getSuccess());
+ assertEquals(OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION,
+ response.getOMResponse().getStatus());
+ assertFalse(omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName))
+ .hasVersioningStatus());
+ }
+
+ @Test
+ public void testVersioningStatusTransitions() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+
+ // UNVERSIONED -> ENABLED
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED)).validateAndUpdateCache(ozoneManager, 1);
+ assertTrue(response.getOMResponse().getSuccess());
+ OmBucketInfo dbBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(BucketVersioningStatus.ENABLED, dbBucketInfo.getVersioningStatus());
+ assertTrue(dbBucketInfo.getIsVersionEnabled());
+
+ // ENABLED -> SUSPENDED
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.SUSPENDED)).validateAndUpdateCache(ozoneManager, 2);
+ assertTrue(response.getOMResponse().getSuccess());
+ dbBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(BucketVersioningStatus.SUSPENDED, dbBucketInfo.getVersioningStatus());
+ assertFalse(dbBucketInfo.getIsVersionEnabled());
+
+ // SUSPENDED -> UNVERSIONED is rejected
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.UNVERSIONED)).validateAndUpdateCache(ozoneManager, 3);
+ assertFalse(response.getOMResponse().getSuccess());
+ assertEquals(OzoneManagerProtocolProtos.Status.INVALID_REQUEST,
+ response.getOMResponse().getStatus());
+ assertThat(response.getOMResponse().getMessage())
+ .contains("once enabled, versioning can only be suspended");
+ assertEquals(BucketVersioningStatus.SUSPENDED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+
+ // SUSPENDED -> ENABLED
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED)).validateAndUpdateCache(ozoneManager, 4);
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(BucketVersioningStatus.ENABLED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+ }
+
+ /**
+ * Changing the versioning status is a bucket property change, so it takes
+ * that request's permission: under native ACLs, the bucket owner or an
+ * administrator.
+ */
+ @Test
+ public void testOnlyOwnerOrAdminMayChangeVersioningStatus()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.getBucketOwner(eq(volumeName), eq(bucketName),
+ any(IAccessAuthorizer.ACLType.class), any(OzoneObj.ResourceType.class)))
+ .thenReturn("bucketOwner");
+
+ // neither the owner nor an administrator
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class)))
+ .thenReturn(false);
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), anyString()))
+ .thenReturn(false);
+ OMClientResponse response = setVersioningStatusAs("regularUser",
+ volumeName, bucketName, BucketVersioningStatus.ENABLED, 1);
+ assertEquals(OzoneManagerProtocolProtos.Status.PERMISSION_DENIED,
+ response.getOMResponse().getStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+
+ // the bucket owner
+ when(ozoneManager.isOwner(any(UserGroupInformation.class),
+ eq("bucketOwner"))).thenReturn(true);
+ response = setVersioningStatusAs("bucketOwner",
+ volumeName, bucketName, BucketVersioningStatus.ENABLED, 2);
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(BucketVersioningStatus.ENABLED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+
+ // an administrator who does not own the bucket
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), anyString()))
+ .thenReturn(false);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class)))
+ .thenReturn(true);
+ response = setVersioningStatusAs("adminUser",
+ volumeName, bucketName, BucketVersioningStatus.SUSPENDED, 3);
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(BucketVersioningStatus.SUSPENDED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+ }
+
+ @Test
+ public void testLegacyVersioningFlagMapsToStateMachine() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ // legacy false on a never-enabled bucket stays UNVERSIONED
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningFlagRequest(volumeName, bucketName, false))
+ .validateAndUpdateCache(ozoneManager, 1);
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ omMetadataManager.getBucketTable().get(bucketKey).getVersioningStatus());
+
+ // legacy true does NOT opt the bucket into S3 versioning: it only sets the
+ // legacy flag, which selects the in-record block version list
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningFlagRequest(volumeName, bucketName, true))
+ .validateAndUpdateCache(ozoneManager, 2);
+ assertTrue(response.getOMResponse().getSuccess());
+ OmBucketInfo dbBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
+ assertFalse(dbBucketInfo.hasVersioningStatus());
+ assertEquals(BucketVersioningStatus.UNVERSIONED,
+ dbBucketInfo.getVersioningStatus());
+ assertTrue(dbBucketInfo.getIsVersionEnabled());
+
+ // once a status exists, an old client's flag is kept consistent with it:
+ // disabling maps to SUSPENDED rather than back to UNVERSIONED
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED))
+ .validateAndUpdateCache(ozoneManager, 3);
+ assertTrue(response.getOMResponse().getSuccess());
+
+ response = new OMBucketSetPropertyRequest(
+ createSetVersioningFlagRequest(volumeName, bucketName, false))
+ .validateAndUpdateCache(ozoneManager, 4);
+ assertTrue(response.getOMResponse().getSuccess());
+ dbBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(BucketVersioningStatus.SUSPENDED, dbBucketInfo.getVersioningStatus());
+ assertFalse(dbBucketInfo.getIsVersionEnabled());
+ }
+
+ @Test
+ public void testVersioningStatusRejectedBeforeFinalization()
+ throws Exception {
+ OMRequest request = createSetVersioningStatusRequest(
+ UUID.randomUUID().toString(), UUID.randomUUID().toString(),
+ BucketVersioningStatus.ENABLED);
+
+ OMLayoutVersionManager preFinalizedVersionManager =
+ mock(OMLayoutVersionManager.class);
+ when(preFinalizedVersionManager
+ .isAllowed(OMLayoutFeature.OBJECT_VERSIONING)).thenReturn(false);
+ ValidationContext preFinalizedContext = ValidationContext.of(
+ preFinalizedVersionManager, omMetadataManager);
+
+ OMException omException = assertThrows(OMException.class,
+ () -> OMBucketSetPropertyRequest
+ .disallowSetBucketPropertyWithVersioningStatus(
+ request, preFinalizedContext));
+ assertEquals(OMException.ResultCodes
+ .NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+ omException.getResult());
+
+ // requests without a versioningStatus pass through untouched
+ OMRequest legacyRequest = createSetVersioningFlagRequest(
+ UUID.randomUUID().toString(), UUID.randomUUID().toString(), true);
+ assertSame(legacyRequest, OMBucketSetPropertyRequest
+ .disallowSetBucketPropertyWithVersioningStatus(
+ legacyRequest, preFinalizedContext));
+
+ // after finalization the request passes through untouched
+ OMLayoutVersionManager finalizedVersionManager =
+ mock(OMLayoutVersionManager.class);
+ when(finalizedVersionManager
+ .isAllowed(OMLayoutFeature.OBJECT_VERSIONING)).thenReturn(true);
+ ValidationContext finalizedContext = ValidationContext.of(
+ finalizedVersionManager, omMetadataManager);
+ assertSame(request, OMBucketSetPropertyRequest
+ .disallowSetBucketPropertyWithVersioningStatus(
+ request, finalizedContext));
+ }
+
+ private OMClientResponse setVersioningStatusAs(String user,
+ String volumeName, String bucketName, BucketVersioningStatus status,
+ long trxnLogIndex) throws Exception {
+ OMBucketSetPropertyRequest request = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName, status));
+ request.setUGI(UserGroupInformation.createRemoteUser(user));
+ return request.validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ /**
+ * Versions are reclaimed by the lifecycle scan and by nothing else, so a
+ * bucket must not be able to enter a state where they accumulate with
+ * nothing able to reclaim them. Checked in preExecute, since whether the
+ * service runs is a property of the receiving OM rather than of replicated
+ * state.
+ */
+ @Test
+ public void testEnablingVersioningNeedsTheLifecycleService()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequest request = createSetVersioningStatusRequest(volumeName,
+ bucketName, BucketVersioningStatus.ENABLED);
+
+ withLifecycleService(false);
+ OMException ex = assertThrows(OMException.class,
+ () -> new OMBucketSetPropertyRequest(request).preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION,
+ ex.getResult());
+ assertThat(ex.getMessage()).contains("lifecycle service is disabled");
+
+ withLifecycleService(true);
+ assertNotNull(new OMBucketSetPropertyRequest(request)
+ .preExecute(ozoneManager));
+ }
+
+ /**
+ * Suspending is not blocked: it stops new versions from being created, so
+ * it can only reduce what has to be reclaimed.
+ */
+ @Test
+ public void testSuspendingVersioningNeedsNoLifecycleService()
+ throws Exception {
+ withLifecycleService(false);
+
+ assertNotNull(new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(UUID.randomUUID().toString(),
+ UUID.randomUUID().toString(), BucketVersioningStatus.SUSPENDED))
+ .preExecute(ozoneManager));
+ }
+
+ /**
+ * The full transition matrix while the bucket holds snapshots. Anything that
+ * lands the bucket in a versioned state is refused, because a noncurrent
+ * version reclaimed from the active object store could take blocks a
+ * snapshot still references. ENABLED -> SUSPENDED is the exception: a
+ * bucket that reached a mixed state before these checks existed has to be
+ * able to stop accumulating versions.
+ */
+ @CsvSource({
+ "UNVERSIONED, ENABLED, false",
+ "UNVERSIONED, SUSPENDED, false",
+ "SUSPENDED, ENABLED, false",
+ "ENABLED, SUSPENDED, true",
+ "ENABLED, ENABLED, true",
+ "SUSPENDED, SUSPENDED, true",
+ })
+ @ParameterizedTest
+ public void testVersioningTransitionsWhileSnapshotsExist(
+ BucketVersioningStatus from, BucketVersioningStatus to, boolean allowed)
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ seedVersioningStatus(volumeName, bucketName, from);
+ addSnapshotToDB(volumeName, bucketName, "snap1");
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName, to))
+ .validateAndUpdateCache(ozoneManager, 2);
+
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ if (allowed) {
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(to, omMetadataManager.getBucketTable().get(bucketKey)
+ .getVersioningStatus());
+ } else {
+ assertFalse(response.getOMResponse().getSuccess());
+ assertEquals(OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION,
+ response.getOMResponse().getStatus());
+ assertThat(response.getOMResponse().getMessage())
+ .contains("while it has snapshots");
+ // the bucket is left as it was
+ assertEquals(from, omMetadataManager.getBucketTable().get(bucketKey)
+ .getVersioningStatus());
+ }
+ }
+
+ /** Without snapshots the same transitions are all allowed. */
+ @CsvSource({
+ "UNVERSIONED, ENABLED",
+ "UNVERSIONED, SUSPENDED",
+ "SUSPENDED, ENABLED",
+ })
+ @ParameterizedTest
+ public void testVersioningTransitionsWithoutSnapshots(
+ BucketVersioningStatus from, BucketVersioningStatus to) throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ seedVersioningStatus(volumeName, bucketName, from);
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName, to))
+ .validateAndUpdateCache(ozoneManager, 2);
+
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(to, omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName))
+ .getVersioningStatus());
+ }
+
+ /**
+ * A snapshot that was deleted but not yet purged still references the blocks
+ * it captured, so it has to keep the bucket out of a versioned state. It is
+ * still in the snapshotInfoTable until the purge, which is what the check
+ * looks at; this pins that, since moving the chain removal earlier would
+ * silently weaken it.
+ */
+ @Test
+ public void testDeletedButUnpurgedSnapshotStillBlocksVersioning()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ SnapshotInfo snapshotInfo = addSnapshotToDB(volumeName, bucketName, "snap1");
+ snapshotInfo.setSnapshotStatus(
+ SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED);
+ omMetadataManager.getSnapshotInfoTable()
+ .put(snapshotInfo.getTableKey(), snapshotInfo);
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED))
+ .validateAndUpdateCache(ozoneManager, 2);
+
+ assertFalse(response.getOMResponse().getSuccess());
+ assertEquals(OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION,
+ response.getOMResponse().getStatus());
+ }
+
+ /**
+ * Two buckets whose names share a prefix must not see each other's
+ * snapshots.
+ */
+ @Test
+ public void testSnapshotOfAnotherBucketDoesNotBlockVersioning()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ // a bucket whose name starts with the first one's name
+ OMRequestTestUtils.addBucketToDB(volumeName, bucketName + "-other",
+ omMetadataManager);
+ addSnapshotToDB(volumeName, bucketName + "-other", "snap1");
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED))
+ .validateAndUpdateCache(ozoneManager, 2);
+
+ assertTrue(response.getOMResponse().getSuccess());
+ }
+
+ /**
+ * The dev-only flag lifts the exclusion so that the integration tests for
+ * snapshot-aware version reclamation can be written against a real
+ * mixed-state bucket.
+ */
+ @Test
+ public void testVersioningAllowedWithSnapshotsUnderDevFlag()
+ throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ addSnapshotToDB(volumeName, bucketName, "snap1");
+ when(ozoneManager.isSnapshotVersioningCoexistenceAllowed())
+ .thenReturn(true);
+
+ OMClientResponse response = new OMBucketSetPropertyRequest(
+ createSetVersioningStatusRequest(volumeName, bucketName,
+ BucketVersioningStatus.ENABLED))
+ .validateAndUpdateCache(ozoneManager, 2);
+
+ assertTrue(response.getOMResponse().getSuccess());
+ assertEquals(BucketVersioningStatus.ENABLED,
+ omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName))
+ .getVersioningStatus());
+ }
+
+ private void seedVersioningStatus(String volumeName, String bucketName,
+ BucketVersioningStatus status) throws Exception {
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketKey)
+ .toBuilder().setVersioningStatus(status).build();
+ omMetadataManager.getBucketTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, bucketInfo));
+ omMetadataManager.getBucketTable().put(bucketKey, bucketInfo);
+ }
+
+ private SnapshotInfo addSnapshotToDB(String volumeName, String bucketName,
+ String snapshotName) throws Exception {
+ SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(volumeName,
+ bucketName, snapshotName, UUID.randomUUID(), Time.now());
+ omMetadataManager.getSnapshotInfoTable()
+ .put(snapshotInfo.getTableKey(), snapshotInfo);
+ return snapshotInfo;
+ }
+
+ private void withLifecycleService(boolean enabled) {
+ KeyLifecycleService lifecycleService = mock(KeyLifecycleService.class);
+ when(lifecycleService.isEnabled()).thenReturn(enabled);
+ KeyManager keyManager = mock(KeyManager.class);
+ when(keyManager.getKeyLifecycleService()).thenReturn(lifecycleService);
+ when(ozoneManager.getKeyManager()).thenReturn(keyManager);
+ }
+
+ private OMRequest createSetVersioningStatusRequest(String volumeName,
+ String bucketName, BucketVersioningStatus status) {
+ return OMRequest.newBuilder().setSetBucketPropertyRequest(
+ SetBucketPropertyRequest.newBuilder().setBucketArgs(
+ BucketArgs.newBuilder().setBucketName(bucketName)
+ .setVolumeName(volumeName)
+ .setVersioningStatus(status.toProto()).build()))
+ .setCmdType(OzoneManagerProtocolProtos.Type.SetBucketProperty)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private OMRequest createSetVersioningFlagRequest(String volumeName,
+ String bucketName, boolean isVersionEnabled) {
+ return OMRequest.newBuilder().setSetBucketPropertyRequest(
+ SetBucketPropertyRequest.newBuilder().setBucketArgs(
+ BucketArgs.newBuilder().setBucketName(bucketName)
+ .setVolumeName(volumeName)
+ .setIsVersionEnabled(isVersionEnabled).build()))
+ .setCmdType(OzoneManagerProtocolProtos.Type.SetBucketProperty)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
@Test
public void testSettingQuotaRetainsReplication() throws Exception {
String volumeName1 = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java
index 167fbc354a3c..f219f23f4077 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java
@@ -74,6 +74,7 @@
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.ScmClient;
+import org.apache.hadoop.ozone.om.VersionIdAllocator;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -154,6 +155,8 @@ public void setup() throws Exception {
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
when(ozoneManager.getConfig()).thenReturn(ozoneConfiguration.getObject(OmConfig.class));
+ when(ozoneManager.getVersionIdAllocator())
+ .thenReturn(new VersionIdAllocator(ozoneConfiguration));
OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
when(lvm.isAllowed(anyString())).thenReturn(true);
when(ozoneManager.getVersionManager()).thenReturn(lvm);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyVersioningRequests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyVersioningRequests.java
new file mode 100644
index 000000000000..8947536aa923
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyVersioningRequests.java
@@ -0,0 +1,1841 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.VersionIdGenerator;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteMarkerResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteMarkerResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyVersion;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.util.Time;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests the S3-compatible versioning behaviour of key writes on a
+ * versioning-enabled OBJECT_STORE bucket: a commit freezes a versionId on the
+ * new current version and keeps the version it overwrote as a noncurrent
+ * version in the versionedKeyTable instead of reclaiming it.
+ */
+public class TestOMKeyVersioningRequests extends OMKeyRequestTests {
+
+ /**
+ * versionIds a request proposes, deliberately unrelated to the transaction
+ * index it is applied at: the id no longer comes from the replication log.
+ */
+ private static final long PROPOSED = 4_000_000L;
+ private static final long LATER_PROPOSED = 5_000_000L;
+
+ @Override
+ public BucketLayout getBucketLayout() {
+ return BucketLayout.OBJECT_STORE;
+ }
+
+ private void setupVersionedBucket() throws Exception {
+ setupVersionedBucket(OzoneConsts.QUOTA_RESET, OzoneConsts.QUOTA_RESET);
+ }
+
+ private void setupVersionedBucket(long quotaInBytes, long quotaInNamespace)
+ throws Exception {
+ setupVersionedBucket(quotaInBytes, quotaInNamespace, 0L);
+ }
+
+ private void setupSuspendedBucket() throws Exception {
+ setupSuspendedBucket(OzoneConsts.QUOTA_RESET, OzoneConsts.QUOTA_RESET, 0L);
+ }
+
+ private void setupSuspendedBucket(long quotaInBytes, long quotaInNamespace,
+ long usedNamespace) throws Exception {
+ setupVersionedBucket(quotaInBytes, quotaInNamespace, usedNamespace,
+ BucketVersioningStatus.SUSPENDED);
+ }
+
+ private void setupVersionedBucket(long quotaInBytes, long quotaInNamespace,
+ long usedNamespace) throws Exception {
+ setupVersionedBucket(quotaInBytes, quotaInNamespace, usedNamespace,
+ BucketVersioningStatus.ENABLED);
+ }
+
+ private void setupVersionedBucket(long quotaInBytes, long quotaInNamespace,
+ long usedNamespace, BucketVersioningStatus status) throws Exception {
+ OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager);
+ OmBucketInfo bucketInfo = OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setVersioningStatus(status)
+ .setQuotaInBytes(quotaInBytes)
+ .setQuotaInNamespace(quotaInNamespace)
+ .setUsedNamespace(usedNamespace)
+ .setCreationTime(Time.now())
+ .build();
+ omMetadataManager.getBucketTable().addCacheEntry(
+ new CacheKey<>(omMetadataManager.getBucketKey(volumeName, bucketName)),
+ CacheValue.get(1L, bucketInfo));
+ }
+
+ /** Puts a current version into keyTable, as an earlier write would have. */
+ private String seedCurrentVersion(Long versionId) throws Exception {
+ return seedCurrentVersion(versionId, false);
+ }
+
+ /** Puts a current version of another key, to batch alongside the first. */
+ private void seedCurrentVersion(String otherKey, Long versionId)
+ throws Exception {
+ seedCurrentVersion(otherKey, versionId, false);
+ }
+
+ private void seedCurrentVersion(String otherKey, Long versionId,
+ boolean deleteMarker) throws Exception {
+ OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, otherKey, replicationConfig)
+ .setVersionId(versionId)
+ .setDeleteMarker(deleteMarker)
+ .build();
+ omMetadataManager.getKeyTable(getBucketLayout()).put(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, otherKey),
+ keyInfo);
+ }
+
+ private String seedCurrentVersion(Long versionId, boolean deleteMarker)
+ throws Exception {
+ return seedCurrentVersion(versionId, deleteMarker, false);
+ }
+
+ private String seedCurrentVersion(Long versionId, boolean deleteMarker,
+ boolean nullVersion) throws Exception {
+ return seedCurrentVersion(versionId, deleteMarker, nullVersion, false);
+ }
+
+ private String seedCurrentVersion(Long versionId, boolean deleteMarker,
+ boolean nullVersion, boolean withBlocks) throws Exception {
+ OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setDeleteMarker(deleteMarker)
+ .setNullVersion(nullVersion)
+ .build();
+ if (withBlocks) {
+ OMRequestTestUtils.addKeyLocationInfo(keyInfo, 0L, 1000L);
+ }
+ String ozoneKey = omMetadataManager.getOzoneKey(
+ volumeName, bucketName, keyName);
+ omMetadataManager.getKeyTable(getBucketLayout()).put(ozoneKey, keyInfo);
+ return ozoneKey;
+ }
+
+ private OMRequest deleteRequest(long proposedVersionId) {
+ KeyArgs keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setModificationTime(Time.now())
+ .setProposedVersionId(proposedVersionId)
+ .build();
+ return OMRequest.newBuilder()
+ .setDeleteKeyRequest(DeleteKeyRequest.newBuilder().setKeyArgs(keyArgs))
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private OMClientResponse deleteAt(long trxnLogIndex, long proposedVersionId)
+ throws Exception {
+ OMClientResponse response =
+ new OMKeyDeleteRequest(deleteRequest(proposedVersionId),
+ getBucketLayout()).validateAndUpdateCache(ozoneManager,
+ trxnLogIndex);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ return response;
+ }
+
+ private OMRequest deleteVersionRequest(Long versionId, boolean nullVersion) {
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setModificationTime(Time.now());
+ if (versionId != null) {
+ keyArgs.setVersionId(versionId);
+ }
+ if (nullVersion) {
+ keyArgs.setNullVersion(true);
+ }
+ return OMRequest.newBuilder()
+ .setDeleteKeyRequest(DeleteKeyRequest.newBuilder().setKeyArgs(keyArgs))
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ /**
+ * S3 names the null version with the literal versionId "null", so a request
+ * that also carries a numeric id names a version twice and is malformed.
+ */
+ @Test
+ public void testDeleteRejectsNamingAVersionTwice() throws Exception {
+ setupVersionedBucket();
+
+ OMException ex = assertThrows(OMException.class,
+ () -> new OMKeyDeleteRequest(deleteVersionRequest(200L, true),
+ getBucketLayout()).preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult());
+ }
+
+ private OMClientResponse deleteVersionAt(Long versionId, boolean nullVersion,
+ long trxnLogIndex) throws Exception {
+ return new OMKeyDeleteRequest(deleteVersionRequest(versionId, nullVersion),
+ getBucketLayout()).validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ private void seedNoncurrentVersion(long versionId, boolean nullVersion)
+ throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setNullVersion(nullVersion)
+ .build();
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId), version);
+ }
+
+ /** A noncurrent version that only exists in the table cache, not in the DB. */
+ private void cacheOnlyNoncurrentVersion(long versionId) throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .build();
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId), version, 350L);
+ }
+
+ @Test
+ public void testPermanentDeleteRemovesOnlyTheAddressedVersion()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+ seedNoncurrentVersion(200L, false);
+
+ OMClientResponse response = deleteVersionAt(100L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ assertNull(noncurrentVersion(100L));
+ assertNotNull(noncurrentVersion(200L));
+ assertEquals(300L, currentVersion().getVersionId());
+ }
+
+ /**
+ * Permanent delete is the one delete on a versioned bucket that destroys
+ * data, so the version's blocks have to reach the deletedTable, which is the
+ * only path that ever reclaims them.
+ */
+ @Test
+ public void testPermanentDeleteSendsTheBlocksToTheDeletedTable()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersionWithBlocks(100L);
+
+ OMClientResponse response = deleteVersionAt(100L, false, 400L);
+ try (BatchOperation batch =
+ omMetadataManager.getStore().initBatchOperation()) {
+ response.checkAndUpdateDB(omMetadataManager, batch);
+ omMetadataManager.getStore().commitBatchOperation(batch);
+ }
+
+ assertNull(omMetadataManager.getVersionedKeyTable().getSkipCache(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, 100L)));
+ assertFalse(deletedVersions().isEmpty(),
+ "the deleted version's blocks never reached the deletedTable");
+ }
+
+ /**
+ * Every version of a key carries the objectID of the record it overwrote.
+ * Named by objectID, two version deletes would land on one deletedTable
+ * entry, and the later put would drop the earlier version's blocks for good.
+ * Deleting the current version twice in a row is the plainest case: each
+ * delete promotes the version under it.
+ */
+ @Test
+ public void testDeletingTheCurrentVersionTwiceKeepsBothInTheDeletedTable()
+ throws Exception {
+ setupVersionedBucket();
+ // A real OM derives the id from the transaction; the mock would answer 0
+ // for every transaction and hide the difference this test is about.
+ when(ozoneManager.getObjectIdFromTxId(anyLong()))
+ .thenAnswer(invocation -> invocation.getArgument(0));
+ seedVersionWithBlocks(300L, true, 42L);
+ seedVersionWithBlocks(200L, false, 42L);
+
+ OMClientResponse first = deleteVersionAt(300L, false, 400L);
+ OMClientResponse second = deleteVersionAt(200L, false, 500L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ first.getOMResponse().getStatus());
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ second.getOMResponse().getStatus());
+ try (BatchOperation batch =
+ omMetadataManager.getStore().initBatchOperation()) {
+ first.checkAndUpdateDB(omMetadataManager, batch);
+ second.checkAndUpdateDB(omMetadataManager, batch);
+ omMetadataManager.getStore().commitBatchOperation(batch);
+ }
+
+ List deleted = new ArrayList<>();
+ for (OmKeyInfo keyInfo : deletedVersions()) {
+ deleted.add(keyInfo.getVersionId());
+ }
+ assertTrue(deleted.contains(300L), "lost the blocks of " + deleted);
+ assertTrue(deleted.contains(200L), "lost the blocks of " + deleted);
+ }
+
+ /** A version holding one block, with the objectID it shares. */
+ private void seedVersionWithBlocks(long versionId, boolean current,
+ long objectId) throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setObjectID(objectId)
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(version, 0L, 100L);
+ if (current) {
+ omMetadataManager.getKeyTable(getBucketLayout()).put(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName),
+ version);
+ } else {
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId), version);
+ }
+ }
+
+ /**
+ * A noncurrent version holding one block. A version with no blocks has
+ * nothing to reclaim and never reaches the deletedTable.
+ */
+ private void seedNoncurrentVersionWithBlocks(long versionId)
+ throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(version, 0L, 100L);
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId), version);
+ }
+
+ /** Every record the deletedTable holds for this key. */
+ private List deletedVersions() throws Exception {
+ List deleted = new ArrayList<>();
+ try (Table.KeyValueIterator entries =
+ omMetadataManager.getDeletedTable().iterator()) {
+ while (entries.hasNext()) {
+ Table.KeyValue entry = entries.next();
+ if (entry.getKey().contains(keyName)) {
+ deleted.addAll(entry.getValue().getOmKeyInfoList());
+ }
+ }
+ }
+ return deleted;
+ }
+
+ @Test
+ public void testPermanentDeleteReleasesQuota() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+
+ OmBucketInfo before = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName));
+ long usedNamespaceBefore = before.getUsedNamespace();
+
+ deleteVersionAt(100L, false, 400L);
+
+ OmBucketInfo after = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName));
+ assertEquals(usedNamespaceBefore - 1, after.getUsedNamespace());
+ }
+
+ @Test
+ public void testPermanentDeleteOfNullVersion() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, true);
+ seedNoncurrentVersion(200L, false);
+
+ OMClientResponse response = deleteVersionAt(null, true, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ assertNull(noncurrentVersion(100L));
+ assertNotNull(noncurrentVersion(200L));
+ }
+
+ /**
+ * Deleting the current version with nothing left to promote takes the key
+ * away entirely, so it comes off the key count. Promotion keeps the key, so
+ * it does not.
+ */
+ @Test
+ public void testDeletingTheLastVersionTakesTheKeyOffTheCount()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, false);
+
+ long before = omMetrics.getNumKeys();
+ deleteVersionAt(300L, false, 400L);
+ // 200 was promoted, so the key is still there
+ assertEquals(before, omMetrics.getNumKeys());
+
+ deleteVersionAt(200L, false, 500L);
+ assertEquals(before - 1, omMetrics.getNumKeys());
+ }
+
+ @Test
+ public void testPermanentDeleteOfUnknownVersionIsNotFound() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+
+ OMClientResponse response = deleteVersionAt(999L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND,
+ response.getOMResponse().getStatus());
+ }
+
+ /**
+ * A bucket with no versioning holds no version an id could name, which is the
+ * same answer as naming an id no version carries: not-found, not unsupported.
+ * The gateway turns it into the plain success S3 gives a delete that names a
+ * version which does not exist.
+ */
+ @Test
+ public void testDeletingAVersionOnAnUnversionedBucketIsNotFound()
+ throws Exception {
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, getBucketLayout());
+ seedCurrentVersion(null);
+
+ OMClientResponse response = deleteVersionAt(300L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND,
+ response.getOMResponse().getStatus());
+ assertNotNull(currentVersion());
+ }
+
+ @Test
+ public void testDeletingCurrentVersionPromotesTheNextNewest()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+ seedNoncurrentVersion(200L, false);
+
+ OMClientResponse response = deleteVersionAt(300L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ // the newest remaining version takes over, unchanged
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertEquals(200L, current.getVersionId());
+ assertFalse(current.isDeleteMarker());
+ // and no longer counts as noncurrent
+ assertNull(noncurrentVersion(200L));
+ assertNotNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * Promotion is positional: the record moves tables and is not rewritten, so
+ * a version keeps the identity it was created with. Comparing the whole
+ * record catches a promotion that rebuilds it - a refreshed updateID or
+ * modification time would silently change what the version means.
+ */
+ @Test
+ public void testPromotedVersionTravelsUnchanged() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ // with blocks, so the location list is part of what has to survive
+ seedNoncurrentVersionWithBlocks(200L);
+ OmKeyInfo before = noncurrentVersion(200L);
+
+ deleteVersionAt(300L, false, 400L);
+
+ assertEquals(before, currentVersion());
+ }
+
+ /**
+ * A version written by a transaction that the double buffer has not flushed
+ * yet lives only in the table cache. Promotion has to see it, otherwise an
+ * older version takes over and the newest one is orphaned.
+ */
+ @Test
+ public void testPromotionSeesVersionsStillInCache() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+ cacheOnlyNoncurrentVersion(200L);
+
+ deleteVersionAt(300L, false, 400L);
+
+ assertEquals(200L, currentVersion().getVersionId());
+ assertNotNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * The mirror case: a version removed by an unflushed transaction is a
+ * tombstone in the cache while the DB still holds it. Promotion must not
+ * bring it back.
+ */
+ @Test
+ public void testPromotionSkipsVersionsTombstonedInCache() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+ seedNoncurrentVersion(200L, false);
+ // 200 is deleted but not flushed yet
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, 200L)),
+ CacheValue.get(350L));
+
+ deleteVersionAt(300L, false, 400L);
+
+ assertEquals(100L, currentVersion().getVersionId());
+ }
+
+ @Test
+ public void testDeletingTheOnlyVersionRemovesTheKey() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+
+ OMClientResponse response = deleteVersionAt(300L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ assertNull(currentVersion());
+ }
+
+ /** Deleting a current delete marker is how S3 restores an object. */
+ @Test
+ public void testDeletingCurrentMarkerRestoresTheObject() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L, true);
+ seedNoncurrentVersion(100L, false);
+
+ OMClientResponse response = deleteVersionAt(300L, false, 400L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertEquals(100L, current.getVersionId());
+ assertFalse(current.isDeleteMarker());
+ }
+
+ private OmKeyInfo currentVersion() throws Exception {
+ return omMetadataManager.getKeyTable(getBucketLayout()).get(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName));
+ }
+
+ private OMRequest commitRequest(boolean isHsync, long proposedVersionId,
+ long dataSize, long writerClientId) {
+ KeyArgs keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setModificationTime(Time.now())
+ .setDataSize(dataSize)
+ .setProposedVersionId(proposedVersionId)
+ .build();
+ return OMRequest.newBuilder()
+ .setCommitKeyRequest(CommitKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setClientID(writerClientId)
+ .setHsync(isHsync))
+ .setCmdType(OzoneManagerProtocolProtos.Type.CommitKey)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private OMClientResponse commitAt(long trxnLogIndex, long proposedVersionId)
+ throws Exception {
+ return commitAt(trxnLogIndex, proposedVersionId, false);
+ }
+
+ private OMClientResponse commitAt(long trxnLogIndex, long proposedVersionId,
+ boolean isHsync) throws Exception {
+ OMClientResponse response =
+ commitAt(trxnLogIndex, proposedVersionId, isHsync, 0L, clientID);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ return response;
+ }
+
+ /**
+ * Commits the key as the writer identified by {@code writerClientId}. A
+ * non-hsync commit tombstones its own open key, so a second write of the
+ * same key comes from a different client, as it would in practice.
+ */
+ private OMClientResponse commitAt(long trxnLogIndex, long proposedVersionId,
+ boolean isHsync, long dataSize, long writerClientId) throws Exception {
+ OMRequestTestUtils.addKeyToTable(true, volumeName, bucketName, keyName,
+ writerClientId, replicationConfig, omMetadataManager);
+ return new OMKeyCommitRequest(
+ commitRequest(isHsync, proposedVersionId, dataSize, writerClientId),
+ getBucketLayout()).validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ private OmKeyInfo noncurrentVersion(long versionId) throws Exception {
+ return omMetadataManager.getVersionedKeyTable().get(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId));
+ }
+
+ @Test
+ public void testCommitOfNewKeyAssignsVersionIdAndHasNoNoncurrentVersion()
+ throws Exception {
+ setupVersionedBucket();
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo current = omMetadataManager.getKeyTable(getBucketLayout()).get(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName));
+ assertNotNull(current);
+ assertEquals(PROPOSED, current.getVersionId());
+ assertFalse(current.isDeleteMarker());
+ assertFalse(current.isNullVersion());
+ assertNull(noncurrentVersion(PROPOSED));
+ }
+
+ /**
+ * The id is proposed on the OM that received the request, so it travels in
+ * the replicated request rather than being generated during apply.
+ */
+ @Test
+ public void testPreExecuteProposesAVersionId() throws Exception {
+ setupVersionedBucket();
+
+ OMRequest processed = new OMKeyCommitRequest(
+ commitRequest(false, VersionIdGenerator.UNSET_VERSION_ID, 0L,
+ clientID),
+ getBucketLayout()).preExecute(ozoneManager);
+
+ assertTrue(processed.getCommitKeyRequest().getKeyArgs()
+ .getProposedVersionId() > VersionIdGenerator.UNSET_VERSION_ID);
+ }
+
+ /**
+ * A proposal is only a floor. A current version numbered above it - which a
+ * clock regression or a change of generator can produce - still has to be
+ * superseded by a larger id, or the versionedKeyTable would order the key's
+ * versions wrongly.
+ */
+ @Test
+ public void testCommitRaisesAProposalBelowTheCurrentVersion()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(PROPOSED + 10);
+
+ commitAt(500L, PROPOSED);
+
+ assertEquals(PROPOSED + 11,
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey)
+ .getVersionId());
+ }
+
+ @Test
+ public void testOverwriteKeepsPreviousVersionAsNoncurrent()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(100L);
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo current =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
+ assertEquals(PROPOSED, current.getVersionId());
+
+ OmKeyInfo noncurrent = noncurrentVersion(100L);
+ assertNotNull(noncurrent);
+ assertEquals(100L, noncurrent.getVersionId());
+ assertFalse(noncurrent.isNullVersion());
+ }
+
+ /**
+ * Both tables are updated in one WriteBatch, so what the table cache shows
+ * during apply is what the DB holds once the batch is committed: a reader
+ * that missed the cache finds the new current version and the version it
+ * superseded, never one without the other.
+ */
+ @Test
+ public void testBothTablesAreWrittenInOneBatch() throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(100L);
+
+ OMClientResponse response = commitAt(500L, PROPOSED);
+ try (BatchOperation batch =
+ omMetadataManager.getStore().initBatchOperation()) {
+ response.checkAndUpdateDB(omMetadataManager, batch);
+ omMetadataManager.getStore().commitBatchOperation(batch);
+ }
+
+ assertEquals(PROPOSED, omMetadataManager.getKeyTable(getBucketLayout())
+ .getSkipCache(ozoneKey).getVersionId());
+ assertNotNull(omMetadataManager.getVersionedKeyTable().getSkipCache(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, 100L)));
+ }
+
+ /**
+ * A NoncurrentVersionExpiration rule counts from the moment a version
+ * stopped being current, so the record is stamped with it as it is moved.
+ * Its own modification time stays put: that is when it was written, which is
+ * a different moment and is what an Expiration rule reads.
+ */
+ @Test
+ public void testOverwriteStampsWhenTheVersionBecameNoncurrent()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(100L);
+ long writtenAt = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(ozoneKey).getModificationTime();
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo current =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
+ OmKeyInfo noncurrent = noncurrentVersion(100L);
+ assertEquals(current.getModificationTime(),
+ noncurrent.getNoncurrentTime());
+ assertEquals(writtenAt, noncurrent.getModificationTime());
+ // the version that superseded it is current, so it has no such moment
+ assertEquals(0L, current.getNoncurrentTime());
+ }
+
+ /**
+ * Promoting a version back to current clears the stamp: it is current
+ * again, so it has no moment of having stopped being current. Left behind,
+ * the stamp would expire the promoted version as if it were still
+ * noncurrent.
+ */
+ @Test
+ public void testPromotingAVersionClearsItsNoncurrentStamp()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(100L);
+
+ commitAt(500L, PROPOSED);
+ assertTrue(noncurrentVersion(100L).getNoncurrentTime() > 0);
+
+ // delete the current version, promoting 100 back into the keyTable
+ deleteVersionAt(PROPOSED, false, 600L);
+
+ OmKeyInfo promoted =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
+ assertEquals(100L, promoted.getVersionId());
+ assertEquals(0L, promoted.getNoncurrentTime());
+ }
+
+ /**
+ * A record written before versioning was enabled carries no versionId, so on
+ * the first overwrite it becomes the key's single null version.
+ */
+ @Test
+ public void testPreVersioningRecordBecomesNullVersion() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(null);
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo noncurrent =
+ noncurrentVersion(VersionIdGenerator.UNSET_VERSION_ID);
+ assertNotNull(noncurrent);
+ assertTrue(noncurrent.isNullVersion());
+ assertEquals(VersionIdGenerator.UNSET_VERSION_ID,
+ noncurrent.getVersionId());
+ }
+
+ /**
+ * The overwritten version keeps its blocks in the versionedKeyTable: they
+ * must not be queued for reclamation, and they must not be carried into the
+ * new current version's in-record block version list either.
+ */
+ @Test
+ public void testOverwriteDoesNotReclaimOrInheritPreviousBlocks()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(100L);
+
+ commitAt(500L, PROPOSED);
+
+ assertNull(omMetadataManager.getDeletedTable().get(ozoneKey));
+ OmKeyInfo current =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
+ assertEquals(1, current.getKeyLocationVersions().size());
+ assertNotNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * An hsync re-commit keeps updating the version it opened rather than
+ * creating a new one, so its versionId stays frozen and nothing moves to the
+ * versionedKeyTable.
+ */
+ @Test
+ public void testHsyncRecommitKeepsVersionIdFrozen() throws Exception {
+ setupVersionedBucket();
+
+ commitAt(500L, PROPOSED, true);
+ OmKeyInfo firstCommit = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName));
+ assertEquals(PROPOSED, firstCommit.getVersionId());
+
+ commitAt(600L, LATER_PROPOSED, true);
+ OmKeyInfo recommitted = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName));
+ assertEquals(PROPOSED, recommitted.getVersionId());
+ assertNull(noncurrentVersion(PROPOSED));
+ }
+
+ @Test
+ public void testDeleteInsertsMarkerAndKeepsPreviousVersion()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+
+ deleteAt(200L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertTrue(current.isDeleteMarker());
+ assertEquals(PROPOSED, current.getVersionId());
+ assertEquals(0L, current.getDataSize());
+ assertTrue(current.getLatestVersionLocations().getLocationList().isEmpty());
+
+ OmKeyInfo noncurrent = noncurrentVersion(100L);
+ assertNotNull(noncurrent);
+ assertFalse(noncurrent.isDeleteMarker());
+ }
+
+ /** A marker is a version, so a delete proposes an id in preExecute too. */
+ @Test
+ public void testDeletePreExecuteProposesAVersionId() throws Exception {
+ setupVersionedBucket();
+
+ OMRequest processed = new OMKeyDeleteRequest(
+ deleteRequest(VersionIdGenerator.UNSET_VERSION_ID), getBucketLayout())
+ .preExecute(ozoneManager);
+
+ assertTrue(processed.getDeleteKeyRequest().getKeyArgs()
+ .getProposedVersionId() > VersionIdGenerator.UNSET_VERSION_ID);
+ }
+
+ /**
+ * A batch delete is the same operation over more keys, so it has to leave
+ * the bucket in the same shape a single delete would: a marker per key, the
+ * version each one superseded kept. Hard-deleting instead would destroy the
+ * current version and strand the versions under it.
+ */
+ @Test
+ public void testBatchDeleteInsertsAMarkerPerKey() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+ String otherKey = keyName + "-other";
+ seedCurrentVersion(otherKey, 150L);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName, otherKey), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ // both keys are now markers, and neither superseded version was reclaimed
+ assertTrue(currentVersion().isDeleteMarker());
+ assertEquals(PROPOSED, currentVersion().getVersionId());
+ assertNotNull(noncurrentVersion(100L));
+
+ OmKeyInfo otherCurrent = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(omMetadataManager.getOzoneKey(volumeName, bucketName, otherKey));
+ assertTrue(otherCurrent.isDeleteMarker());
+ // one proposal covers the batch: ids only have to increase within a key
+ assertEquals(PROPOSED, otherCurrent.getVersionId());
+ assertNotNull(omMetadataManager.getVersionedKeyTable().get(
+ omMetadataManager.getVersionedOzoneKey(volumeName, bucketName,
+ otherKey, 150L)));
+
+ assertInstanceOf(OMKeysDeleteMarkerResponse.class, response);
+ }
+
+ /**
+ * A batch that fails part way through has already put entries into the
+ * versionedKeyTable cache. The double buffer cleans the table cache from the
+ * response's CleanupTableInfo, so the failure response has to declare the
+ * same tables as the successful one or those entries are in no DB and are
+ * never cleaned up.
+ */
+ @Test
+ public void testAFailedBatchDeleteStillCleansTheVersionedKeyTable()
+ throws Exception {
+ // namespace for one more record, so the second marker cannot be written
+ setupVersionedBucket(OzoneConsts.QUOTA_RESET, 3L, 2L);
+ seedCurrentVersion(100L);
+ String otherKey = keyName + "-other";
+ seedCurrentVersion(otherKey, 150L);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName, otherKey), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED,
+ response.getOMResponse().getStatus());
+ assertInstanceOf(OMKeysDeleteMarkerResponse.class, response);
+ // The bucket is the cached instance, which the next write to it persists:
+ // the marker that did fit must not stay counted for a batch that failed.
+ assertEquals(2L, omMetadataManager.getBucketTable().get(bucketKey)
+ .getUsedNamespace());
+ }
+
+ /**
+ * S3 records the delete whether or not the key was there, so a key the
+ * bucket holds no record of takes a delete marker of its own instead of
+ * failing the batch. The single-key delete already does this.
+ */
+ @Test
+ public void testBatchDeleteMarksAKeyThatDoesNotExist() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+ String absentKey = keyName + "-absent";
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName, absentKey), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+
+ // the whole batch succeeds: a missing key is not a partial failure
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ OmKeyInfo marker = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(omMetadataManager.getOzoneKey(volumeName, bucketName, absentKey));
+ assertNotNull(marker);
+ assertTrue(marker.isDeleteMarker());
+ // nothing was superseded, so the marker stands alone
+ assertNull(omMetadataManager.getVersionedKeyTable().get(
+ omMetadataManager.getVersionedOzoneKey(volumeName, bucketName,
+ absentKey, PROPOSED)));
+ }
+
+ /**
+ * Only a key that stopped being visible to a plain read comes off the key
+ * count. A marker written over a key that was already a marker, or over one
+ * the bucket held no record of, hides nothing that was not hidden already.
+ */
+ @Test
+ public void testBatchDeleteCountsOnlyTheKeysItHides() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+ String markedKey = keyName + "-marked";
+ seedCurrentVersion(markedKey, 150L, true);
+ String absentKey = keyName + "-absent";
+
+ long before = omMetrics.getNumKeys();
+ new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName, markedKey, absentKey),
+ getBucketLayout()).validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(before - 1, omMetrics.getNumKeys());
+ }
+
+ /** The proposal is a floor for each key independently. */
+ @Test
+ public void testBatchDeleteRaisesTheProposalPerKey() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(PROPOSED + 10);
+
+ new OMKeysDeleteRequest(batchDeleteRequest(PROPOSED, keyName),
+ getBucketLayout()).validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(PROPOSED + 11, currentVersion().getVersionId());
+ }
+
+ /** A bucket without versioning keeps the old hard-delete behaviour. */
+ @Test
+ public void testBatchDeleteOnAnUnversionedBucketStillDeletes()
+ throws Exception {
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, getBucketLayout());
+ seedCurrentVersion(null);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNull(currentVersion());
+ }
+
+ /**
+ * S3 calls the object itself the null version on a bucket that has never
+ * been versioned, so addressing it is the plain delete rather than an
+ * unsupported request.
+ */
+ @Test
+ public void testDeleteByNullVersionOnAnUnversionedBucketDeletes()
+ throws Exception {
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, getBucketLayout());
+ seedCurrentVersion(null);
+
+ OMClientResponse response = deleteVersionAt(null, true, 200L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNull(currentVersion());
+ }
+
+ /**
+ * A marker that takes the key's null version slot replaces a record instead
+ * of adding one, so a bucket sitting exactly at its namespace quota still
+ * accepts the delete.
+ */
+ @Test
+ public void testSuspendedMarkerAtNamespaceQuotaReplacesTheNullVersion()
+ throws Exception {
+ // the one record the key holds is all the quota allows
+ setupSuspendedBucket(OzoneConsts.QUOTA_RESET, 1L, 1L);
+ seedCurrentVersion(100L, false, true);
+
+ OMClientResponse response =
+ new OMKeyDeleteRequest(deleteRequest(PROPOSED), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertTrue(currentVersion().isDeleteMarker());
+ assertEquals(1, omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName))
+ .getUsedNamespace());
+ }
+
+ /**
+ * A marker keeps the key's identity and none of its content, so what it
+ * carries is listed rather than inherited from the version it supersedes.
+ */
+ @Test
+ public void testDeleteMarkerKeepsIdentityAndDropsContent() throws Exception {
+ setupVersionedBucket();
+ OmKeyInfo seeded = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(100L)
+ .setOwnerName("owner1")
+ .setAcls(Collections.singletonList(
+ OzoneAcl.parseAcl("user:bilbo:rwdlncxy[ACCESS]")))
+ .addMetadata("meta", "value")
+ .addTag("tag", "value")
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(seeded, 0L, 1000L);
+ String ozoneKey =
+ omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
+ omMetadataManager.getKeyTable(getBucketLayout()).put(ozoneKey, seeded);
+
+ deleteAt(200L, PROPOSED);
+
+ OmKeyInfo marker = currentVersion();
+ assertTrue(marker.isDeleteMarker());
+ assertEquals(seeded.getObjectID(), marker.getObjectID());
+ assertEquals("owner1", marker.getOwnerName());
+ assertEquals(seeded.getReplicationConfig(),
+ marker.getReplicationConfig());
+ assertEquals(1, marker.getAcls().size());
+
+ assertEquals(0L, marker.getDataSize());
+ assertTrue(marker.getMetadata().isEmpty());
+ assertTrue(marker.getTags().isEmpty());
+ assertTrue(marker.getKeyLocationVersions().get(0)
+ .getLocationList().isEmpty());
+ }
+
+ private OMRequest batchDeleteRequest(long proposedVersionId, String... keys) {
+ return OMRequest.newBuilder()
+ .setDeleteKeysRequest(DeleteKeysRequest.newBuilder()
+ .setDeleteKeys(DeleteKeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllKeys(java.util.Arrays.asList(keys))
+ .setProposedVersionId(proposedVersionId)))
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private OMRequest batchDeleteRequest(List keys, KeyVersion... versions) {
+ return OMRequest.newBuilder()
+ .setDeleteKeysRequest(DeleteKeysRequest.newBuilder()
+ .setDeleteKeys(DeleteKeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllKeys(keys)
+ .addAllKeyVersions(Arrays.asList(versions))
+ .setProposedVersionId(PROPOSED)))
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private OMClientResponse batchDeleteAt(long trxnLogIndex, List keys, KeyVersion... versions) {
+ return new OMKeysDeleteRequest(batchDeleteRequest(keys, versions), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ private KeyVersion version(long versionId) {
+ return KeyVersion.newBuilder().setKey(keyName).setVersionId(versionId).build();
+ }
+
+ private void commit(OMClientResponse response) throws Exception {
+ try (BatchOperation batch = omMetadataManager.getStore().initBatchOperation()) {
+ response.checkAndUpdateDB(omMetadataManager, batch);
+ omMetadataManager.getStore().commitBatchOperation(batch);
+ }
+ }
+
+ /**
+ * A batch can name several versions of one key, and those share a
+ * deletedTable key; the blocks of each have to reach it.
+ */
+ @Test
+ public void testBatchDeleteOfTwoVersionsOfOneKeyKeepsBothInTheDeletedTable() throws Exception {
+ setupVersionedBucket();
+ // see testDeletingTheCurrentVersionTwiceKeepsBothInTheDeletedTable
+ when(ozoneManager.getObjectIdFromTxId(anyLong()))
+ .thenAnswer(invocation -> invocation.getArgument(0));
+ seedCurrentVersion(300L);
+ seedVersionWithBlocks(100L, false, 42L);
+ seedVersionWithBlocks(200L, false, 42L);
+
+ OMClientResponse response = batchDeleteAt(400L, Collections.emptyList(), version(100L), version(200L));
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+ commit(response);
+
+ assertNull(noncurrentVersion(100L));
+ assertNull(noncurrentVersion(200L));
+ assertEquals(300L, currentVersion().getVersionId());
+ List deleted = new ArrayList<>();
+ for (OmKeyInfo keyInfo : deletedVersions()) {
+ deleted.add(keyInfo.getVersionId());
+ }
+ assertTrue(deleted.contains(100L), "lost the blocks of " + deleted);
+ assertTrue(deleted.contains(200L), "lost the blocks of " + deleted);
+ }
+
+ @Test
+ public void testBatchDeleteOfTheCurrentVersionPromotesTheNextNewest() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, false);
+
+ OMClientResponse response = batchDeleteAt(400L, Collections.emptyList(), version(300L));
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+
+ assertEquals(200L, currentVersion().getVersionId());
+ assertNull(noncurrentVersion(200L));
+ }
+
+ @Test
+ public void testBatchDeleteOfTheNullVersion() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, true);
+
+ OMClientResponse response = batchDeleteAt(400L, Collections.emptyList(),
+ KeyVersion.newBuilder().setKey(keyName).setNullVersion(true).build());
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+
+ assertNull(noncurrentVersion(100L));
+ assertEquals(300L, currentVersion().getVersionId());
+ }
+
+ /** A version that does not exist fails its own entry, not the batch. */
+ @Test
+ public void testBatchDeleteReportsAMissingVersionOnItsOwn() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+
+ OMClientResponse response = batchDeleteAt(400L, Collections.emptyList(), version(100L), version(999L));
+ assertEquals(OzoneManagerProtocolProtos.Status.PARTIAL_DELETE, response.getOMResponse().getStatus());
+ assertNull(noncurrentVersion(100L));
+
+ DeleteKeysResponse result = response.getOMResponse().getDeleteKeysResponse();
+ assertEquals(1, result.getErrorsCount());
+ assertEquals(999L, result.getErrors(0).getVersionId());
+ assertEquals(OMException.ResultCodes.KEY_NOT_FOUND.name(), result.getErrors(0).getErrorCode());
+ assertEquals(Collections.singletonList(version(999L)), result.getUnDeletedKeys().getKeyVersionsList());
+ }
+
+ /**
+ * A key named both plainly and by a version is marked first, so the version
+ * delete finds the version the marker superseded among the noncurrent ones.
+ */
+ @Test
+ public void testBatchDeleteMarksAKeyBeforeDeletingAVersionOfIt() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+
+ OMClientResponse response = batchDeleteAt(400L, Collections.singletonList(keyName), version(300L));
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+ commit(response);
+
+ assertTrue(currentVersion().isDeleteMarker());
+ assertNull(omMetadataManager.getVersionedKeyTable().getSkipCache(
+ omMetadataManager.getVersionedOzoneKey(volumeName, bucketName, keyName, 300L)));
+ }
+
+ /**
+ * A plain key that fails on a versioned bucket is reported, rather than taken for deleted, while the other keys
+ * still get their delete markers.
+ */
+ @Test
+ public void testBatchDeleteReportsAPlainKeyThatFailsOnAVersionedBucket() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(300L);
+ String deniedKey = "denied-key";
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(Arrays.asList(keyName, deniedKey)), getBucketLayout()) {
+ @Override
+ protected void checkKeyAcls(OzoneManager om, String volume, String bucket, String key,
+ IAccessAuthorizer.ACLType aclType, OzoneObj.ResourceType resourceType, String volumeOwner)
+ throws IOException {
+ if (deniedKey.equals(key)) {
+ throw new OMException("denied", OMException.ResultCodes.PERMISSION_DENIED);
+ }
+ }
+ }.validateAndUpdateCache(ozoneManager, 400L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.PARTIAL_DELETE, response.getOMResponse().getStatus());
+ List errors = response.getOMResponse().getDeleteKeysResponse().getErrorsList();
+ assertEquals(1, errors.size());
+ assertEquals(deniedKey, errors.get(0).getKey());
+ assertEquals(OMException.ResultCodes.ACCESS_DENIED.name(), errors.get(0).getErrorCode());
+ commit(response);
+ assertTrue(currentVersion().isDeleteMarker());
+ }
+
+ @Test
+ public void testBatchDeleteRejectsAnEntryNamingNoVersionOrBoth() {
+ for (KeyVersion entry : Arrays.asList(
+ KeyVersion.newBuilder().setKey(keyName).build(),
+ version(300L).toBuilder().setNullVersion(true).build())) {
+ OMException ex = assertThrows(OMException.class,
+ () -> new OMKeysDeleteRequest(batchDeleteRequest(Collections.emptyList(), entry), getBucketLayout())
+ .preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult());
+ }
+ }
+
+ /** S3 inserts a delete marker even for a key that does not exist. */
+ @Test
+ public void testDeleteOfMissingKeyStillInsertsMarker() throws Exception {
+ setupVersionedBucket();
+
+ deleteAt(200L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertTrue(current.isDeleteMarker());
+ assertEquals(PROPOSED, current.getVersionId());
+ assertNull(noncurrentVersion(VersionIdGenerator.UNSET_VERSION_ID));
+ }
+
+ /** Deleting a key whose current version is already a marker stacks another. */
+ @Test
+ public void testDeleteStacksAnotherMarker() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L, true);
+
+ deleteAt(200L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertTrue(current.isDeleteMarker());
+ assertEquals(PROPOSED, current.getVersionId());
+
+ OmKeyInfo stacked = noncurrentVersion(100L);
+ assertNotNull(stacked);
+ assertTrue(stacked.isDeleteMarker());
+ }
+
+ @Test
+ public void testDeleteOfPreVersioningRecordMovesItToNullVersion()
+ throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(null);
+
+ deleteAt(200L, PROPOSED);
+
+ OmKeyInfo noncurrent =
+ noncurrentVersion(VersionIdGenerator.UNSET_VERSION_ID);
+ assertNotNull(noncurrent);
+ assertTrue(noncurrent.isNullVersion());
+ assertFalse(noncurrent.isDeleteMarker());
+ }
+
+ /**
+ * Every version counts against the bucket's space quota: an overwrite adds
+ * the new version's usage without releasing the version it supersedes.
+ */
+ @Test
+ public void testEachVersionCountsAgainstUsedBytes() throws Exception {
+ setupVersionedBucket();
+ String bucketKey =
+ omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ commitAt(500L, PROPOSED, false, 100L, clientID).getOMResponse().getStatus());
+ long afterFirst = omMetadataManager.getBucketTable().get(bucketKey)
+ .getUsedBytes();
+ assertEquals(QuotaUtil.getReplicatedSize(100L, replicationConfig),
+ afterFirst);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ commitAt(600L, LATER_PROPOSED, false, 300L, clientID + 1).getOMResponse().getStatus());
+ long afterSecond = omMetadataManager.getBucketTable().get(bucketKey)
+ .getUsedBytes();
+ assertEquals(afterFirst
+ + QuotaUtil.getReplicatedSize(300L, replicationConfig),
+ afterSecond);
+ assertEquals(2, omMetadataManager.getBucketTable().get(bucketKey)
+ .getUsedNamespace());
+ }
+
+ @Test
+ public void testVersionedWriteRejectedWhenSpaceQuotaExceeded()
+ throws Exception {
+ long quota = QuotaUtil.getReplicatedSize(150L, replicationConfig);
+ setupVersionedBucket(quota, OzoneConsts.QUOTA_RESET);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ commitAt(500L, PROPOSED, false, 100L, clientID).getOMResponse().getStatus());
+ // the first version is not released, so the second one no longer fits
+ assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED,
+ commitAt(600L, LATER_PROPOSED, false, 100L, clientID + 1)
+ .getOMResponse().getStatus());
+
+ OmKeyInfo current = currentVersion();
+ assertEquals(PROPOSED, current.getVersionId());
+ assertNull(noncurrentVersion(PROPOSED));
+ }
+
+ @Test
+ public void testVersionedWriteRejectedWhenNamespaceQuotaExceeded()
+ throws Exception {
+ setupVersionedBucket(OzoneConsts.QUOTA_RESET, 1L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ commitAt(500L, PROPOSED, false, 0L, clientID).getOMResponse().getStatus());
+ // each version is a record of its own, so the second one needs namespace
+ assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED,
+ commitAt(600L, LATER_PROPOSED, false, 0L, clientID + 1).getOMResponse().getStatus());
+ }
+
+ /** A delete marker is a record too, so it needs namespace quota. */
+ @Test
+ public void testDeleteMarkerRejectedWhenNamespaceQuotaExceeded()
+ throws Exception {
+ // the bucket already holds the one key its namespace quota allows
+ setupVersionedBucket(OzoneConsts.QUOTA_RESET, 1L, 1L);
+ seedCurrentVersion(100L);
+
+ OMClientResponse response =
+ new OMKeyDeleteRequest(deleteRequest(PROPOSED), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+ assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED,
+ response.getOMResponse().getStatus());
+ assertFalse(currentVersion().isDeleteMarker());
+ // a rejected request must not leave the superseded version behind in the
+ // versionedKeyTable cache, and its response has to declare that table so
+ // that the double buffer cleans up whatever the request did touch
+ assertNull(noncurrentVersion(100L));
+ assertInstanceOf(OMKeyDeleteMarkerResponse.class, response);
+ }
+
+ /** Entries naming the same version are kept once, however each names it. */
+ @Test
+ public void testPreExecuteDropsAVersionNamedTwice() throws Exception {
+ OMRequest request = new OMKeysDeleteRequest(batchDeleteRequest(Collections.emptyList(), version(1L),
+ version(1L).toBuilder().setNullVersion(false).build()), getBucketLayout()).preExecute(ozoneManager);
+ assertEquals(Collections.singletonList(version(1L)),
+ request.getDeleteKeysRequest().getDeleteKeys().getKeyVersionsList());
+ }
+
+ /**
+ * A key named twice in one batch is deleted once: one marker, counted once against the namespace quota.
+ */
+ @Test
+ public void testBatchDeleteOfAKeyNamedTwiceInsertsOneMarker() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ long usedNamespace = omMetadataManager.getBucketTable().get(bucketKey).getUsedNamespace();
+
+ OMRequest request = new OMKeysDeleteRequest(batchDeleteRequest(Arrays.asList(keyName, keyName)), getBucketLayout())
+ .preExecute(ozoneManager);
+ assertEquals(1, request.getDeleteKeysRequest().getDeleteKeys().getKeysCount());
+ OMClientResponse response = new OMKeysDeleteRequest(request, getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 200L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+ assertEquals(usedNamespace + 1, omMetadataManager.getBucketTable().get(bucketKey).getUsedNamespace());
+ }
+
+ /**
+ * A delete marker holds no blocks, so it consumes namespace but no space,
+ * and the superseded version keeps its own usage.
+ */
+ @Test
+ public void testDeleteMarkerConsumesNamespaceButNoSpace() throws Exception {
+ setupVersionedBucket();
+ seedCurrentVersion(100L);
+ String bucketKey =
+ omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo before = omMetadataManager.getBucketTable().get(bucketKey);
+ long usedBytes = before.getUsedBytes();
+ long usedNamespace = before.getUsedNamespace();
+
+ deleteAt(200L, PROPOSED);
+
+ OmBucketInfo after = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(usedBytes, after.getUsedBytes());
+ assertEquals(usedNamespace + 1, after.getUsedNamespace());
+ }
+
+ /**
+ * A write while versioning is suspended takes the key's null version slot
+ * instead of creating a version of its own.
+ */
+ @Test
+ public void testSuspendedWriteTakesTheNullVersionSlot() throws Exception {
+ setupSuspendedBucket();
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertTrue(current.isNullVersion());
+ assertEquals(PROPOSED, current.getVersionId());
+ }
+
+ /**
+ * Repeated suspended writes replace each other: versions do not accumulate,
+ * and the replaced record's blocks are queued for reclamation.
+ */
+ @Test
+ public void testSuspendedWriteReplacesTheCurrentNullVersion()
+ throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(100L, false, true, true);
+
+ OMKeyCommitResponse response =
+ (OMKeyCommitResponse) commitAt(500L, PROPOSED);
+
+ assertTrue(currentVersion().isNullVersion());
+ assertEquals(PROPOSED, currentVersion().getVersionId());
+ // the replaced null version is not kept as a noncurrent version
+ assertNull(noncurrentVersion(100L));
+ // its blocks are queued for reclamation instead
+ assertReclaimed(response, 100L);
+ }
+
+ /** Asserts that the given version was queued for block reclamation. */
+ private void assertReclaimed(OMKeyCommitResponse response, long versionId) {
+ assertNotNull(response.getKeysToDelete());
+ assertTrue(response.getKeysToDelete().values().stream()
+ .flatMap(repeated -> repeated.getOmKeyInfoList().stream())
+ .anyMatch(info -> info.getVersionId() != null
+ && info.getVersionId() == versionId),
+ "version " + versionId + " was not queued for reclamation");
+ }
+
+ /**
+ * Versions created while versioning was enabled are not touched by a
+ * suspended write: the one it supersedes becomes noncurrent as usual.
+ */
+ @Test
+ public void testSuspendedWriteKeepsEnabledEraVersions() throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(100L, false);
+
+ commitAt(500L, PROPOSED);
+
+ assertTrue(currentVersion().isNullVersion());
+ // the version it superseded is retained, as is the older one
+ assertNotNull(noncurrentVersion(300L));
+ assertFalse(noncurrentVersion(300L).isNullVersion());
+ assertNotNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * The null version lookup walks the versionedKeyTable cache and opens an
+ * iterator on the DB, inline on the write path, so its cost is reported.
+ */
+ @Test
+ public void testTheNullVersionLookupIsTimed() throws Exception {
+ OMPerformanceMetrics perf = mock(OMPerformanceMetrics.class);
+ when(ozoneManager.getPerfMetrics()).thenReturn(perf);
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, true);
+
+ commitAt(500L, PROPOSED);
+
+ verify(perf, atLeastOnce()).addNoncurrentVersionLookupLatency(anyLong());
+ }
+
+ /**
+ * The null version may be noncurrent, when versioning was enabled again
+ * after the write that created it. A suspended write still replaces it, and
+ * still becomes the current version.
+ */
+ @Test
+ public void testSuspendedWriteReplacesANoncurrentNullVersion()
+ throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, true);
+ seedNoncurrentVersion(100L, false);
+
+ commitAt(500L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertTrue(current.isNullVersion());
+ assertEquals(PROPOSED, current.getVersionId());
+ // the old null version is gone, everything else is retained
+ assertNull(noncurrentVersion(200L));
+ assertNotNull(noncurrentVersion(300L));
+ assertNotNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * Replacing the key's null version leaves the bucket's record count
+ * unchanged, so the quota check must not see the record the write adds
+ * without the one it removes: a bucket sitting exactly at its namespace
+ * quota still accepts the write.
+ */
+ @Test
+ public void testSuspendedWriteAtNamespaceQuotaReplacesTheNullVersion()
+ throws Exception {
+ // the two records the key already holds are all the quota allows
+ setupSuspendedBucket(OzoneConsts.QUOTA_RESET, 2L, 2L);
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, true);
+
+ OMClientResponse response = commitAt(500L, PROPOSED, false, 0L, clientID);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNull(noncurrentVersion(200L));
+ assertEquals(2, omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName))
+ .getUsedNamespace());
+ }
+
+ /**
+ * A delete while versioning is suspended writes a marker into the key's null
+ * version slot rather than creating a version.
+ */
+ @Test
+ public void testSuspendedDeleteWritesANullMarker() throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+
+ deleteAt(500L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertNotNull(current);
+ assertTrue(current.isDeleteMarker());
+ assertTrue(current.isNullVersion());
+ // the version it superseded is retained, as under an enabled bucket
+ assertNotNull(noncurrentVersion(300L));
+ }
+
+ /** The null marker replaces the null version that held the slot. */
+ @Test
+ public void testSuspendedDeleteReplacesTheCurrentNullVersion()
+ throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(100L, false, true, true);
+
+ OMKeyDeleteMarkerResponse response =
+ (OMKeyDeleteMarkerResponse) deleteAt(500L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertTrue(current.isDeleteMarker());
+ assertTrue(current.isNullVersion());
+ // the replaced record is not kept as a noncurrent version
+ assertNull(noncurrentVersion(100L));
+ assertNotNull(response.getKeysToDelete());
+ }
+
+ /**
+ * Versions created while versioning was enabled stay readable and deletable
+ * by versionId after a suspended delete.
+ */
+ @Test
+ public void testSuspendedDeleteKeepsEnabledEraVersions() throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+ seedNoncurrentVersion(200L, true);
+ seedNoncurrentVersion(100L, false);
+
+ deleteAt(500L, PROPOSED);
+
+ assertTrue(currentVersion().isDeleteMarker());
+ // the superseded version and the older one are retained
+ assertNotNull(noncurrentVersion(300L));
+ assertNotNull(noncurrentVersion(100L));
+ // only the null version the marker replaced is gone
+ assertNull(noncurrentVersion(200L));
+
+ // and a retained version can still be deleted by versionId
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ deleteVersionAt(100L, false, 600L).getOMResponse().getStatus());
+ assertNull(noncurrentVersion(100L));
+ }
+
+ /**
+ * A batch delete on a suspended bucket takes the same path as a single one.
+ * Hard-deleting there would destroy the current version and strand the
+ * versions written while versioning was enabled, which is the failure a
+ * suspended bucket is explicitly not allowed to have: suspending stops new
+ * versions from being created, it does not stop the old ones from being
+ * protected.
+ */
+ @Test
+ public void testSuspendedBatchDeleteWritesANullMarker() throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertInstanceOf(OMKeysDeleteMarkerResponse.class, response);
+
+ OmKeyInfo current = currentVersion();
+ assertTrue(current.isDeleteMarker());
+ assertTrue(current.isNullVersion());
+ assertNotNull(noncurrentVersion(300L));
+ }
+
+ /**
+ * Suspending stops new versions from being created, not the delete from
+ * being recorded, so a key the bucket holds no record of takes a null
+ * delete marker here too.
+ */
+ @Test
+ public void testSuspendedBatchDeleteMarksAKeyThatDoesNotExist()
+ throws Exception {
+ setupSuspendedBucket();
+ String absentKey = keyName + "-absent";
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, absentKey), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ OmKeyInfo marker = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(omMetadataManager.getOzoneKey(volumeName, bucketName, absentKey));
+ assertNotNull(marker);
+ assertTrue(marker.isDeleteMarker());
+ assertTrue(marker.isNullVersion());
+ }
+
+ /**
+ * The null slot the batch marker replaces is dropped from the DB and its
+ * blocks queued, like the single-key path. Leaving the record behind would
+ * keep a version nothing can address; skipping the queue would leak its
+ * blocks after the quota for them was already returned.
+ */
+ @Test
+ public void testSuspendedBatchDeleteReplacesTheNullVersion()
+ throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(300L);
+ seedNullVersionWithBlocks(200L);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 500L);
+ assertInstanceOf(OMKeysDeleteMarkerResponse.class, response);
+
+ try (BatchOperation batch =
+ omMetadataManager.getStore().initBatchOperation()) {
+ response.checkAndUpdateDB(omMetadataManager, batch);
+ omMetadataManager.getStore().commitBatchOperation(batch);
+ }
+
+ assertTrue(currentVersion().isDeleteMarker());
+ assertTrue(currentVersion().isNullVersion());
+ // the null version is gone from the DB, not only from the cache
+ assertNull(omMetadataManager.getVersionedKeyTable().getSkipCache(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, 200L)));
+ assertFalse(deletedVersions().isEmpty(),
+ "the replaced null version's blocks never reached the deletedTable");
+ // the version written while versioning was enabled is untouched
+ assertNotNull(omMetadataManager.getVersionedKeyTable().getSkipCache(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, 300L)));
+ }
+
+ /** A noncurrent null version holding one block. */
+ /**
+ * A marker that takes the null version slot releases that version's space.
+ * The bucket is the cached instance, so a batch that then fails on a later
+ * key has to leave the space where it found it.
+ */
+ @Test
+ public void testAFailedSuspendedBatchLeavesTheBucketsSpaceAlone()
+ throws Exception {
+ // namespace is full: replacing the null slot fits, the next marker does not
+ setupSuspendedBucket(OzoneConsts.QUOTA_RESET, 2L, 2L);
+ seedCurrentVersion(300L);
+ seedNullVersionWithBlocks(200L);
+ String otherKey = keyName + "-other";
+ seedCurrentVersion(otherKey, 150L);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ // get() hands back a copy; the request works on the cached instance
+ omMetadataManager.getBucketTable().getCacheValue(new CacheKey<>(bucketKey))
+ .getCacheValue().incrUsedBytes(1000L);
+ long usedBytesBefore =
+ omMetadataManager.getBucketTable().get(bucketKey).getUsedBytes();
+ assertEquals(1000L, usedBytesBefore);
+
+ OMClientResponse response = new OMKeysDeleteRequest(
+ batchDeleteRequest(PROPOSED, keyName, otherKey), getBucketLayout())
+ .validateAndUpdateCache(ozoneManager, 500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED,
+ response.getOMResponse().getStatus());
+ assertEquals(usedBytesBefore,
+ omMetadataManager.getBucketTable().get(bucketKey).getUsedBytes());
+ }
+
+ private void seedNullVersionWithBlocks(long versionId) throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setNullVersion(true)
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(version, 0L, 100L);
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName, versionId), version);
+ }
+
+ /**
+ * A key written before versioning was enabled carries no versionId and no
+ * null flag. It is the key's null version all the same, so version "null"
+ * addresses it - without the record ever having been rewritten.
+ */
+ @Test
+ public void testPreVersioningCurrentIsAddressableAsNullVersion()
+ throws Exception {
+ setupVersionedBucket();
+ String ozoneKey = seedCurrentVersion(null);
+ OmKeyInfo legacy = currentVersion();
+ assertNull(legacy.getVersionId());
+ assertFalse(legacy.isNullVersion());
+ assertTrue(legacy.isNullVersionRecord());
+
+ // deleting version "null" hits it even though it carries no flag
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ deleteVersionAt(null, true, 500L).getOMResponse().getStatus());
+ assertNull(omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey));
+ }
+
+ /**
+ * A suspended write replaces a pre-versioning record, since that record is
+ * the key's null version.
+ */
+ @Test
+ public void testSuspendedWriteReplacesAPreVersioningRecord()
+ throws Exception {
+ setupSuspendedBucket();
+ seedCurrentVersion(null, false, false, true);
+
+ OMKeyCommitResponse response = (OMKeyCommitResponse) commitAt(500L, PROPOSED);
+
+ OmKeyInfo current = currentVersion();
+ assertTrue(current.isNullVersion());
+ assertEquals(PROPOSED, current.getVersionId());
+ // it was replaced, not kept as a noncurrent version
+ assertNull(noncurrentVersion(VersionIdGenerator.UNSET_VERSION_ID));
+ assertNotNull(response.getKeysToDelete());
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java
index 52626c68802b..3c6e338d68e7 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java
@@ -38,7 +38,9 @@
import java.util.List;
import java.util.UUID;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.VersionIdAllocator;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -172,6 +174,10 @@ public void testPreExecuteAdminCheckForPiggybackedScanState() throws Exception {
OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class);
when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+ // preExecute proposes a versionId for the delete markers a versioned
+ // bucket would need, before it knows which bucket the request is for.
+ when(ozoneManager.getVersionIdAllocator())
+ .thenReturn(new VersionIdAllocator(new OzoneConfiguration()));
when(ozoneManager.isAdminAuthorizationEnabled()).thenReturn(true);
when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMObjectVersionsReclaimRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMObjectVersionsReclaimRequest.java
new file mode 100644
index 000000000000..d159caab912b
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMObjectVersionsReclaimRequest.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.response.key.OMObjectVersionsReclaimResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionRecord;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ObjectVersionsBucket;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ReclaimObjectVersionsRequest;
+import org.apache.hadoop.util.Time;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests the reclamation the lifecycle scan submits: noncurrent versions leave
+ * the versionedKeyTable with their blocks queued in the deletedTable, expired
+ * delete markers leave the keyTable, and the bucket quota is released for
+ * both.
+ */
+public class TestOMObjectVersionsReclaimRequest extends OMKeyRequestTests {
+
+ private static final long BLOCK_LENGTH = 1000L;
+
+ @Override
+ public BucketLayout getBucketLayout() {
+ return BucketLayout.OBJECT_STORE;
+ }
+
+ @Test
+ public void testReclaimsSubmittedVersions() throws Exception {
+ setupVersionedBucket(3 * BLOCK_LENGTH, 3L);
+ seedNoncurrentVersion(100L);
+ seedNoncurrentVersion(200L);
+ seedNoncurrentVersion(300L);
+
+ OMObjectVersionsReclaimResponse response = reclaim(500L,
+ versionKeys(100L, 200L));
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+
+ // the reclaimed versions are tombstoned in the versionedKeyTable cache
+ assertReclaimedFromTable(100L);
+ assertReclaimedFromTable(200L);
+ // the version that was not submitted is untouched
+ assertNull(omMetadataManager.getVersionedKeyTable().getCacheValue(
+ new CacheKey<>(versionKey(300L))));
+
+ // both are queued for block reclamation
+ List queued = response.getKeysToDelete().values().stream()
+ .flatMap(repeated -> repeated.getOmKeyInfoList().stream())
+ .map(OmKeyInfo::getVersionId)
+ .collect(Collectors.toList());
+ assertEquals(Arrays.asList(100L, 200L), queued);
+
+ // and their space and namespace are released
+ OmBucketInfo bucketInfo = OMKeyRequest.getBucketInfo(omMetadataManager,
+ volumeName, bucketName);
+ assertEquals(BLOCK_LENGTH, bucketInfo.getUsedBytes());
+ assertEquals(1L, bucketInfo.getUsedNamespace());
+ }
+
+ /**
+ * Versions of one key share a deletedTable entry: it holds a
+ * RepeatedOmKeyInfo list that KeyDeletingService evaluates one record at a
+ * time.
+ */
+ @Test
+ public void testVersionsOfOneKeyShareADeletedTableEntry() throws Exception {
+ setupVersionedBucket(3 * BLOCK_LENGTH, 3L);
+ seedNoncurrentVersion(100L);
+ seedNoncurrentVersion(200L);
+
+ OMObjectVersionsReclaimResponse response = reclaim(500L,
+ versionKeys(100L, 200L));
+
+ assertEquals(1, response.getKeysToDelete().size());
+ assertEquals(2, response.getKeysToDelete().values().iterator().next()
+ .getOmKeyInfoList().size());
+ }
+
+ /**
+ * The service selects versions as of its own scan, so one may already be
+ * gone - permanently deleted, or promoted into the keyTable because the
+ * current version was deleted. Such a version is not this request's to
+ * remove.
+ */
+ @Test
+ public void testSkipsVersionsThatAreAlreadyGone() throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ seedNoncurrentVersion(100L);
+
+ OMObjectVersionsReclaimResponse response = reclaim(500L,
+ versionKeys(100L, 200L));
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertReclaimedFromTable(100L);
+ // only the version that still existed was reclaimed and accounted for
+ assertEquals(1, response.getKeysToDelete().values().stream()
+ .mapToInt(repeated -> repeated.getOmKeyInfoList().size()).sum());
+ OmBucketInfo bucketInfo = OMKeyRequest.getBucketInfo(omMetadataManager,
+ volumeName, bucketName);
+ assertEquals(0L, bucketInfo.getUsedBytes());
+ assertEquals(0L, bucketInfo.getUsedNamespace());
+ }
+
+ @Test
+ public void testSkipsBucketThatNoLongerExists() throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ seedNoncurrentVersion(100L);
+
+ OMRequest request = reclaimRequest(ObjectVersionsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName("deleted-bucket")
+ .addVersions(versionRecord(versionKey(100L)))
+ .build());
+ OMObjectVersionsReclaimResponse response =
+ (OMObjectVersionsReclaimResponse) new OMObjectVersionsReclaimRequest(
+ request).validateAndUpdateCache(ozoneManager, 500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertTrue(response.getKeysToDelete().isEmpty());
+ // the version of the bucket that does exist is left alone
+ assertNull(omMetadataManager.getVersionedKeyTable().getCacheValue(
+ new CacheKey<>(versionKey(100L))));
+ }
+
+ /**
+ * A marker with nothing under it hides nothing and cannot be addressed by
+ * versionId, so nothing else would ever remove it. It leaves the keyTable
+ * and the key disappears with it, releasing the namespace slot it held.
+ */
+ @Test
+ public void testReclaimsAMarkerWithNothingUnderIt() throws Exception {
+ setupVersionedBucket(0L, 1L);
+ seedCurrentMarker(300L);
+
+ OMObjectVersionsReclaimResponse response = reclaimMarkers(500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ CacheValue cached = omMetadataManager
+ .getKeyTable(getBucketLayout()).getCacheValue(new CacheKey<>(ozoneKey()));
+ assertNotNull(cached, "the marker was not reclaimed");
+ assertNull(cached.getCacheValue(), "the marker was not tombstoned");
+
+ // it holds no blocks, so nothing is queued and only namespace is released
+ assertTrue(response.getKeysToDelete().isEmpty());
+ OmBucketInfo bucketInfo = OMKeyRequest.getBucketInfo(omMetadataManager,
+ volumeName, bucketName);
+ assertEquals(0L, bucketInfo.getUsedNamespace());
+ }
+
+ /**
+ * While a noncurrent version survives, removing the marker would promote it
+ * and bring back an object the user deleted. The scan checked this too, but
+ * a version can be written between the scan and this request, so the check
+ * is what happens under the bucket lock that decides.
+ */
+ @Test
+ public void testKeepsAMarkerThatStillHidesAVersion() throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 2L);
+ seedCurrentMarker(300L);
+ seedNoncurrentVersion(100L);
+
+ OMObjectVersionsReclaimResponse response = reclaimMarkers(500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNull(omMetadataManager.getKeyTable(getBucketLayout())
+ .getCacheValue(new CacheKey<>(ozoneKey())));
+ OmBucketInfo bucketInfo = OMKeyRequest.getBucketInfo(omMetadataManager,
+ volumeName, bucketName);
+ assertEquals(2L, bucketInfo.getUsedNamespace());
+ }
+
+ /**
+ * A write since the scan supersedes the marker: the key's current version is
+ * a real object again, and there is nothing expired to remove.
+ */
+ @Test
+ public void testSkipsAMarkerSupersededByAWrite() throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ OmKeyInfo rewritten = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(400L)
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(rewritten, 0L, BLOCK_LENGTH);
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .put(ozoneKey(), rewritten);
+
+ OMObjectVersionsReclaimResponse response = reclaimMarkers(500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNull(omMetadataManager.getKeyTable(getBucketLayout())
+ .getCacheValue(new CacheKey<>(ozoneKey())));
+ }
+
+ private String ozoneKey() {
+ return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
+ }
+
+ /** A key whose current version is a delete marker. */
+ private void seedCurrentMarker(long versionId) throws Exception {
+ OmKeyInfo marker = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setDeleteMarker(true)
+ .build();
+ omMetadataManager.getKeyTable(getBucketLayout()).put(ozoneKey(), marker);
+ }
+
+ private OMObjectVersionsReclaimResponse reclaimMarkers(long trxnLogIndex)
+ throws Exception {
+ return reclaimMarkers(trxnLogIndex, markerRecord(ozoneKey()));
+ }
+
+ private OMObjectVersionsReclaimResponse reclaimMarkers(long trxnLogIndex,
+ ObjectVersionRecord marker) throws Exception {
+ OMRequest request = reclaimRequest(ObjectVersionsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addMarkers(marker)
+ .build());
+ return (OMObjectVersionsReclaimResponse)
+ new OMObjectVersionsReclaimRequest(request)
+ .validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ /** A record naming what sits at the dbKey right now. */
+ private ObjectVersionRecord versionRecord(String dbKey) throws Exception {
+ OmKeyInfo version = omMetadataManager.getVersionedKeyTable().get(dbKey);
+ return record(dbKey, version == null ? 0L : version.getUpdateID());
+ }
+
+ private ObjectVersionRecord markerRecord(String dbKey) throws Exception {
+ OmKeyInfo marker =
+ omMetadataManager.getKeyTable(getBucketLayout()).get(dbKey);
+ return record(dbKey, marker == null ? 0L : marker.getUpdateID());
+ }
+
+ private static ObjectVersionRecord record(String dbKey, long updateId) {
+ return ObjectVersionRecord.newBuilder()
+ .setDbKey(dbKey)
+ .setUpdateId(updateId)
+ .build();
+ }
+
+ /**
+ * A marker's dbKey is the plain key name, so it names whichever version of
+ * the key is current. Deleting the key, writing it and deleting it again
+ * leaves another marker there, which every other check here accepts: it is
+ * a marker, and nothing survives under it. Only the updateID tells it from
+ * the one the scan selected.
+ */
+ @Test
+ public void testAMarkerReplacedSinceTheScanIsNotReclaimed()
+ throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ seedCurrentMarker(300L);
+
+ // the scan read a marker that has since been replaced
+ OMObjectVersionsReclaimResponse response =
+ reclaimMarkers(500L, record(ozoneKey(), 4711L));
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNotNull(
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey()),
+ "a marker the scan never selected was reclaimed");
+ }
+
+ /**
+ * A versionId is allocated against the key's current version alone, so one
+ * that was permanently deleted can be handed out again, putting a different
+ * version at the dbKey the scan selected.
+ */
+ @Test
+ public void testAVersionReplacedSinceTheScanIsNotReclaimed()
+ throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ seedNoncurrentVersion(100L);
+
+ OMRequest request = reclaimRequest(ObjectVersionsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addVersions(record(versionKey(100L), 4711L))
+ .build());
+ OMObjectVersionsReclaimResponse response =
+ (OMObjectVersionsReclaimResponse) new OMObjectVersionsReclaimRequest(
+ request).validateAndUpdateCache(ozoneManager, 500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNotNull(
+ omMetadataManager.getVersionedKeyTable().get(versionKey(100L)),
+ "a version the scan never selected was reclaimed");
+ }
+
+ private void setupVersionedBucket(long usedBytes, long usedNamespace)
+ throws Exception {
+ OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager);
+ OmBucketInfo bucketInfo = OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setVersioningStatus(BucketVersioningStatus.ENABLED)
+ .setQuotaInBytes(OzoneConsts.QUOTA_RESET)
+ .setQuotaInNamespace(OzoneConsts.QUOTA_RESET)
+ .setUsedBytes(usedBytes)
+ .setUsedNamespace(usedNamespace)
+ .setCreationTime(Time.now())
+ .build();
+ omMetadataManager.getBucketTable().addCacheEntry(
+ new CacheKey<>(omMetadataManager.getBucketKey(volumeName, bucketName)),
+ CacheValue.get(1L, bucketInfo));
+ }
+
+ /**
+ * A version demoted by a transaction the double buffer has not flushed yet
+ * lives in the table cache alone, and a prefix iteration over the DB reads
+ * straight past it. Removing the marker then promotes that version back to
+ * current, resurrecting an object the user deleted.
+ */
+ @Test
+ public void testAMarkerIsKeptForAVersionOnlyInTheCache() throws Exception {
+ setupVersionedBucket(BLOCK_LENGTH, 1L);
+ seedCurrentMarker(300L);
+ cacheOnlyNoncurrentVersion(100L);
+
+ OMObjectVersionsReclaimResponse response = reclaimMarkers(500L);
+
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ response.getOMResponse().getStatus());
+ assertNotNull(
+ omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey()),
+ "the marker was reclaimed while a version survived in the cache");
+ }
+
+ /** A noncurrent version that only exists in the table cache, not the DB. */
+ private void cacheOnlyNoncurrentVersion(long versionId) throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .build();
+ OMRequestTestUtils.addKeyLocationInfo(version, 0L, BLOCK_LENGTH);
+ omMetadataManager.getVersionedKeyTable()
+ .addCacheEntry(versionKey(versionId), version, 350L);
+ }
+
+ private void seedNoncurrentVersion(long versionId) throws Exception {
+ seedNoncurrentVersion(versionId, false);
+ }
+
+ private void seedNoncurrentVersion(long versionId, boolean deleteMarker)
+ throws Exception {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setVersionId(versionId)
+ .setDeleteMarker(deleteMarker)
+ .build();
+ if (!deleteMarker) {
+ OMRequestTestUtils.addKeyLocationInfo(version, 0L, BLOCK_LENGTH);
+ }
+ omMetadataManager.getVersionedKeyTable()
+ .put(versionKey(versionId), version);
+ }
+
+ private String versionKey(long versionId) {
+ return omMetadataManager.getVersionedOzoneKey(volumeName, bucketName,
+ keyName, versionId);
+ }
+
+ private List versionKeys(long... versionIds) {
+ return Arrays.stream(versionIds).mapToObj(this::versionKey)
+ .collect(Collectors.toList());
+ }
+
+ private void assertReclaimedFromTable(long versionId) {
+ CacheValue cached = omMetadataManager.getVersionedKeyTable()
+ .getCacheValue(new CacheKey<>(versionKey(versionId)));
+ assertNotNull(cached, "version " + versionId + " was not reclaimed");
+ assertNull(cached.getCacheValue(),
+ "version " + versionId + " was not tombstoned");
+ }
+
+ private OMObjectVersionsReclaimResponse reclaim(long trxnLogIndex,
+ List keys) throws Exception {
+ List records = new ArrayList<>();
+ for (String key : keys) {
+ records.add(versionRecord(key));
+ }
+ OMRequest request = reclaimRequest(ObjectVersionsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllVersions(records)
+ .build());
+ return (OMObjectVersionsReclaimResponse)
+ new OMObjectVersionsReclaimRequest(request)
+ .validateAndUpdateCache(ozoneManager, trxnLogIndex);
+ }
+
+ private OMRequest reclaimRequest(ObjectVersionsBucket versionsBucket) {
+ return OMRequest.newBuilder()
+ .setReclaimObjectVersionsRequest(
+ ReclaimObjectVersionsRequest.newBuilder()
+ .addVersionsPerBucket(versionsBucket))
+ .setCmdType(OzoneManagerProtocolProtos.Type.ReclaimObjectVersions)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
index 08e8487ee087..f257ea07c261 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.ozone.om.OmMetadataReader;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
+import org.apache.hadoop.ozone.om.VersionIdAllocator;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
@@ -122,6 +123,8 @@ public void setup() throws Exception {
when(lvm.getMetadataLayoutVersion()).thenReturn(0);
when(ozoneManager.getVersionManager()).thenReturn(lvm);
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
+ when(ozoneManager.getVersionIdAllocator())
+ .thenReturn(new VersionIdAllocator(ozoneConfiguration));
when(ozoneManager.getConfig()).thenReturn(ozoneConfiguration.getObject(OmConfig.class));
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java
index 27b7217b2dcb..af4624e635fd 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java
@@ -20,9 +20,11 @@
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
import java.io.IOException;
import java.util.ArrayList;
@@ -35,6 +37,8 @@
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
@@ -99,6 +103,111 @@ public void testValidateAndUpdateCacheSuccess() throws Exception {
checkDeleteTableCount(volumeName, bucketName, keyName, 1, uploadId);
}
+
+ /**
+ * Completing a multipart upload over an existing key on a versioned bucket
+ * creates a version like any other write: the version it supersedes has to
+ * be kept in the versionedKeyTable, not dropped from the keyTable and left
+ * out of the deletedTable.
+ */
+ @Test
+ public void testVersionedOverwriteKeepsPreviousVersion() throws Exception {
+ // versioning is only supported on OBJECT_STORE buckets
+ assumeFalse(getBucketLayout().isFileSystemOptimized());
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, omMetadataManager,
+ OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ // versioning is only allowed on OBJECT_STORE buckets; it shares
+ // the keyTable with LEGACY, so the request helpers still apply
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setVersioningStatus(BucketVersioningStatus.ENABLED));
+
+ checkValidateAndUpdateCacheSuccess(volumeName, bucketName, keyName,
+ new HashMap<>(), new HashMap<>(), 0L, 1L);
+ OmKeyInfo firstVersion = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getOzoneDBKey(volumeName, bucketName, keyName));
+ assertNotNull(firstVersion.getVersionId());
+
+ checkValidateAndUpdateCacheSuccess(volumeName, bucketName, keyName,
+ new HashMap<>(), new HashMap<>(), 10L, 2L);
+
+ OmKeyInfo current = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getOzoneDBKey(volumeName, bucketName, keyName));
+ assertNotEquals(firstVersion.getVersionId(), current.getVersionId());
+
+ // the superseded version survives as a noncurrent version
+ OmKeyInfo noncurrent = omMetadataManager.getVersionedKeyTable().get(
+ omMetadataManager.getVersionedOzoneKey(volumeName, bucketName, keyName,
+ firstVersion.getVersionId()));
+ assertNotNull(noncurrent,
+ "the version the upload superseded was neither kept nor reclaimed");
+ assertEquals(firstVersion.getVersionId(), noncurrent.getVersionId());
+
+ // and its blocks were not queued for reclamation: the version record still
+ // refers to them, so reclaiming them would strand it
+ assertEquals(0, deletedEntriesFor(volumeName, bucketName, keyName),
+ "the superseded version's blocks were queued for deletion");
+ }
+
+ /**
+ * A completion while versioning is suspended follows the same rule a
+ * suspended PUT does: the new version takes the key's null slot, replacing
+ * whatever held it, so versions do not accumulate.
+ */
+ @Test
+ public void testSuspendedCompletionTakesTheNullSlot() throws Exception {
+ assumeFalse(getBucketLayout().isFileSystemOptimized());
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = getKeyName();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, omMetadataManager,
+ OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setVersioningStatus(BucketVersioningStatus.SUSPENDED));
+
+ checkValidateAndUpdateCacheSuccess(volumeName, bucketName, keyName,
+ new HashMap<>(), new HashMap<>(), 0L, 1L);
+ OmKeyInfo firstNull = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getOzoneDBKey(volumeName, bucketName, keyName));
+ assertTrue(firstNull.isNullVersion());
+
+ checkValidateAndUpdateCacheSuccess(volumeName, bucketName, keyName,
+ new HashMap<>(), new HashMap<>(), 10L, 1L);
+
+ OmKeyInfo current = omMetadataManager.getKeyTable(getBucketLayout())
+ .get(getOzoneDBKey(volumeName, bucketName, keyName));
+ assertTrue(current.isNullVersion());
+ assertNotEquals(firstNull.getVersionId(), current.getVersionId());
+
+ // the record it replaced is not kept as a noncurrent version
+ assertNull(omMetadataManager.getVersionedKeyTable().get(
+ omMetadataManager.getVersionedOzoneKey(volumeName, bucketName, keyName,
+ firstNull.getVersionId())),
+ "a suspended completion accumulated a version instead of replacing "
+ + "the null slot");
+ }
+
+ /** Records the deletedTable holds for this key, however they were filed. */
+ private int deletedEntriesFor(String volumeName, String bucketName,
+ String keyName) throws Exception {
+ int found = 0;
+ try (Table.KeyValueIterator entries =
+ omMetadataManager.getDeletedTable().iterator()) {
+ while (entries.hasNext()) {
+ if (entries.next().getKey().contains(keyName)) {
+ found++;
+ }
+ }
+ }
+ return found;
+ }
+
public void checkDeleteTableCount(String volumeName,
String bucketName, String keyName, int count, String uploadId)
throws Exception {
@@ -123,6 +232,21 @@ public void checkDeleteTableCount(String volumeName,
private String checkValidateAndUpdateCacheSuccess(String volumeName,
String bucketName, String keyName, Map metadata, Map tags) throws Exception {
+ return checkValidateAndUpdateCacheSuccess(volumeName, bucketName, keyName,
+ metadata, tags, 0L, getNamespaceCount());
+ }
+
+ /**
+ * @param trxnBase offset added to the transaction indexes, so that a test can
+ * run the flow more than once: versionIds have to increase within a key.
+ * @param expectedNamespace the bucket's used namespace once the upload
+ * completes; a versioned overwrite adds a record rather than replacing
+ * one, so the count grows.
+ */
+ private String checkValidateAndUpdateCacheSuccess(String volumeName,
+ String bucketName, String keyName, Map metadata,
+ Map tags, long trxnBase, long expectedNamespace)
+ throws Exception {
OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName,
bucketName, keyName, metadata, tags);
@@ -131,7 +255,7 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName,
getS3InitiateMultipartUploadReq(initiateMPURequest);
OMClientResponse omClientResponse =
- s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, 1L);
+ s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, trxnBase + 1L);
long clientID = Time.now();
String multipartUploadID = omClientResponse.getOMResponse()
@@ -146,7 +270,7 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName,
// Add key to open key table.
addKeyToTable(volumeName, bucketName, keyName, clientID);
- s3MultipartUploadCommitPartRequest.validateAndUpdateCache(ozoneManager, 2L);
+ s3MultipartUploadCommitPartRequest.validateAndUpdateCache(ozoneManager, trxnBase + 2L);
List partList = new ArrayList<>();
@@ -167,7 +291,7 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName,
getS3MultipartUploadCompleteReq(completeMultipartRequest);
omClientResponse =
- s3MultipartUploadCompleteRequest.validateAndUpdateCache(ozoneManager, 3L);
+ s3MultipartUploadCompleteRequest.validateAndUpdateCache(ozoneManager, trxnBase + 3L);
BatchOperation batchOperation
= omMetadataManager.getStore().initBatchOperation();
@@ -202,8 +326,7 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName,
.getCacheValue(new CacheKey<>(
omMetadataManager.getBucketKey(volumeName, bucketName)))
.getCacheValue();
- assertEquals(getNamespaceCount(),
- omBucketInfo.getUsedNamespace());
+ assertEquals(expectedNamespace, omBucketInfo.getUsedNamespace());
return multipartUploadID;
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
index 605e1c10df2a..8a69fce7a828 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
@@ -40,11 +40,14 @@
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
@@ -58,11 +61,13 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
/**
@@ -157,6 +162,115 @@ public void testPreExecuteBadOwner() {
omException.getMessage());
}
+ /**
+ * A snapshot shares blocks with the active object store and keeps them only
+ * because KeyDeletingService refuses to reclaim what a previous snapshot
+ * references. That decision is not version-aware yet, so the two features
+ * cannot be used on the same bucket. SUSPENDED is refused along with ENABLED
+ * because a suspended bucket still holds noncurrent versions.
+ */
+ @EnumSource(value = BucketVersioningStatus.class,
+ names = {"ENABLED", "SUSPENDED"})
+ @ParameterizedTest
+ public void testCreateSnapshotRejectedOnVersionedBucket(
+ BucketVersioningStatus status) throws Exception {
+ when(getOzoneManager().isAdmin(any())).thenReturn(true);
+ setBucketVersioningStatus(getVolumeName(), getBucketName(), status);
+
+ OMRequest omRequest = createSnapshotRequest(getVolumeName(),
+ getBucketName(), snapshotName1);
+ OMSnapshotCreateRequest request = doPreExecute(omRequest);
+
+ OMResponse omResponse = request
+ .validateAndUpdateCache(getOzoneManager(), 1).getOMResponse();
+
+ assertEquals(Status.NOT_SUPPORTED_OPERATION, omResponse.getStatus());
+ assertThat(omResponse.getMessage())
+ .contains("Snapshots and S3 object versioning cannot be used on the"
+ + " same bucket");
+ // nothing was written
+ assertNull(getOmMetadataManager().getSnapshotInfoTable()
+ .get(getTableKey(getVolumeName(), getBucketName(), snapshotName1)));
+ }
+
+ @Test
+ public void testCreateSnapshotAllowedOnUnversionedBucket() throws Exception {
+ when(getOzoneManager().isAdmin(any())).thenReturn(true);
+ setBucketVersioningStatus(getVolumeName(), getBucketName(),
+ BucketVersioningStatus.UNVERSIONED);
+
+ OMRequest omRequest = createSnapshotRequest(getVolumeName(),
+ getBucketName(), snapshotName1);
+ OMSnapshotCreateRequest request = doPreExecute(omRequest);
+
+ assertEquals(OK, request.validateAndUpdateCache(getOzoneManager(), 1)
+ .getOMResponse().getStatus());
+ }
+
+ /**
+ * preExecute resolves a linked bucket to its source, so the check applies to
+ * what the link points at rather than to the link itself.
+ */
+ @Test
+ public void testCreateSnapshotRejectedOnLinkToVersionedBucket()
+ throws Exception {
+ when(getOzoneManager().isAdmin(any())).thenReturn(true);
+ String sourceBucketName = getBucketName() + "-source";
+ OMRequestTestUtils.addVolumeAndBucketToDB(getVolumeName(),
+ sourceBucketName, getOmMetadataManager());
+ setBucketVersioningStatus(getVolumeName(), sourceBucketName,
+ BucketVersioningStatus.ENABLED);
+ when(getOzoneManager().resolveBucketLink(any(Pair.class),
+ any(OMClientRequest.class)))
+ .thenAnswer(i -> new ResolvedBucket(i.getArgument(0),
+ Pair.of(getVolumeName(), sourceBucketName), "owner",
+ BucketLayout.OBJECT_STORE));
+
+ OMRequest omRequest = createSnapshotRequest(getVolumeName(),
+ getBucketName(), snapshotName1);
+ OMSnapshotCreateRequest request = doPreExecute(omRequest);
+
+ OMResponse omResponse = request
+ .validateAndUpdateCache(getOzoneManager(), 1).getOMResponse();
+
+ assertEquals(Status.NOT_SUPPORTED_OPERATION, omResponse.getStatus());
+ assertThat(omResponse.getMessage()).contains(sourceBucketName);
+ }
+
+ /**
+ * The dev-only flag lifts the exclusion so that the integration tests for
+ * snapshot-aware version reclamation can be written against a real
+ * mixed-state bucket.
+ */
+ @Test
+ public void testCreateSnapshotAllowedOnVersionedBucketWithDevFlag()
+ throws Exception {
+ when(getOzoneManager().isAdmin(any())).thenReturn(true);
+ when(getOzoneManager().isSnapshotVersioningCoexistenceAllowed())
+ .thenReturn(true);
+ setBucketVersioningStatus(getVolumeName(), getBucketName(),
+ BucketVersioningStatus.ENABLED);
+
+ OMRequest omRequest = createSnapshotRequest(getVolumeName(),
+ getBucketName(), snapshotName1);
+ OMSnapshotCreateRequest request = doPreExecute(omRequest);
+
+ assertEquals(OK, request.validateAndUpdateCache(getOzoneManager(), 1)
+ .getOMResponse().getStatus());
+ }
+
+ private void setBucketVersioningStatus(String volume, String bucket,
+ BucketVersioningStatus status) throws Exception {
+ String bucketKey = getOmMetadataManager().getBucketKey(volume, bucket);
+ OmBucketInfo bucketInfo = getOmMetadataManager().getBucketTable()
+ .get(bucketKey).toBuilder()
+ .setVersioningStatus(status)
+ .build();
+ getOmMetadataManager().getBucketTable().addCacheEntry(
+ new CacheKey<>(bucketKey), CacheValue.get(1L, bucketInfo));
+ getOmMetadataManager().getBucketTable().put(bucketKey, bucketInfo);
+ }
+
@Test
public void testValidateAndUpdateCache() throws Exception {
when(getOzoneManager().isAdmin(any())).thenReturn(true);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
index 012123680f60..eee5d2509d80 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -99,6 +99,7 @@
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.FaultInjectorImpl;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -110,6 +111,7 @@
import org.apache.hadoop.ozone.om.TrashOzoneFileSystem;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
@@ -2959,6 +2961,142 @@ void testOrphanMpuNotAbortedByTagOnlyRule() throws Exception {
deleteLifecyclePolicy(volumeName, bucketName);
}
+ /**
+ * Expiring an object on a versioned bucket hides it behind a delete
+ * marker rather than deleting it, as S3 does. The version that was
+ * current when the rule matched is demoted, not reclaimed: nothing
+ * reaches the deletedTable and no space is freed, so the run reports a
+ * marker inserted rather than a key deleted.
+ */
+ @Test
+ void testExpirationInsertsADeleteMarkerOnAVersionedBucket()
+ throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialMarkers = metrics.getNumDeleteMarkersInserted().value();
+ long initialKeyDeleted = metrics.getNumKeyDeleted().value();
+
+ createVersionedVolumeAndBucket(volumeName, bucketName);
+ OmKeyArgs key = createAndCommitKey(volumeName, bucketName,
+ uniqueObjectName("key"), 1, null);
+
+ ZonedDateTime date =
+ ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, OBJECT_STORE, "key", null,
+ date.toString(), true);
+
+ GenericTestUtils.waitFor(
+ () -> isDeleteMarker(volumeName, bucketName, key.getKeyName()),
+ WAIT_CHECK_INTERVAL, 10000);
+
+ // the version the marker superseded is kept, addressable by versionId
+ assertEquals(1, countNoncurrentVersions(volumeName, bucketName,
+ key.getKeyName()));
+ // and it was not reclaimed: no blocks were queued, no space released
+ assertEquals(initialDeletedKeyCount, getDeletedKeyCount());
+ assertEquals(initialKeyDeleted, metrics.getNumKeyDeleted().value());
+ assertEquals(initialMarkers + 1,
+ metrics.getNumDeleteMarkersInserted().value());
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ /**
+ * The marker itself must not expire again. A marker is the key's current
+ * version and carries the time it was written, so it ages into the same
+ * rule; putting another marker on top of it would demote the old one and
+ * grow the version chain for as long as the rule exists. Removing an
+ * expired marker is ExpiredObjectDeleteMarker's decision instead.
+ */
+ @Test
+ void testExpirationLeavesAnExistingDeleteMarkerAlone() throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialMarkers = metrics.getNumDeleteMarkersInserted().value();
+
+ createVersionedVolumeAndBucket(volumeName, bucketName);
+ // sorts first, so the scan decides about it before the live key
+ String markerKey = "key-a-already-a-marker";
+ OmKeyInfo marker = seedDeleteMarker(volumeName, bucketName, markerKey);
+ OmKeyArgs liveKey = createAndCommitKey(volumeName, bucketName,
+ "key-b-live", 1, null);
+
+ ZonedDateTime date =
+ ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, OBJECT_STORE, "key", null,
+ date.toString(), true);
+
+ GenericTestUtils.waitFor(
+ () -> isDeleteMarker(volumeName, bucketName, liveKey.getKeyName()),
+ WAIT_CHECK_INTERVAL, 10000);
+
+ // the pre-existing marker matched the rule too, and was left as it was
+ OmKeyInfo stillThere = metadataManager.getKeyTable(OBJECT_STORE)
+ .get(metadataManager.getOzoneKey(volumeName, bucketName, markerKey));
+ assertTrue(stillThere.isDeleteMarker());
+ assertEquals(marker.getVersionId(), stillThere.getVersionId());
+ assertEquals(marker.getModificationTime(),
+ stillThere.getModificationTime());
+ // nothing was demoted under it, so no marker was stacked on it
+ assertEquals(0,
+ countNoncurrentVersions(volumeName, bucketName, markerKey));
+ // only the live key produced a marker
+ assertEquals(initialMarkers + 1,
+ metrics.getNumDeleteMarkersInserted().value());
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+
+ /**
+ * ExpiredObjectDeleteMarker removes a delete marker, and the key with it,
+ * once nothing is left under it - and only then. A marker that still
+ * hides a noncurrent version has to stay: removing it would promote that
+ * version and bring back an object the user deleted.
+ */
+ @Test
+ void testExpiredObjectDeleteMarkerRemovesOnlyALoneMarker()
+ throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+
+ long initialSent = om.getDeletionMetrics()
+ .getNumObjectVersionsSentForReclaim().value();
+
+ createVersionedVolumeAndBucket(volumeName, bucketName);
+ String loneMarker = "key-a-lone-marker";
+ String hidingMarker = "key-b-hides-a-version";
+ seedDeleteMarker(volumeName, bucketName, loneMarker);
+ seedDeleteMarker(volumeName, bucketName, hidingMarker);
+ seedNoncurrentVersion(volumeName, bucketName, hidingMarker, 900L);
+
+ createLifecyclePolicy(volumeName, bucketName, OBJECT_STORE,
+ Collections.singletonList(new OmLCRule.Builder()
+ .setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
+ .setEnabled(true)
+ .setPrefix("key")
+ .setAction(new OmLCExpiration.Builder()
+ .setExpiredObjectDeleteMarker(true)
+ .build())
+ .build()));
+
+ GenericTestUtils.waitFor(
+ () -> !keyExists(volumeName, bucketName, loneMarker),
+ WAIT_CHECK_INTERVAL, 10000);
+
+ // the marker that still hides a version is left exactly as it was
+ assertTrue(isDeleteMarker(volumeName, bucketName, hidingMarker));
+ assertEquals(1, countNoncurrentVersions(volumeName, bucketName,
+ hidingMarker));
+ // and the scan never submitted it: the request refusing it would keep
+ // the marker too, so this is what shows the scan itself ruled it out
+ assertEquals(initialSent + 1, om.getDeletionMetrics()
+ .getNumObjectVersionsSentForReclaim().value());
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
}
/**
@@ -3266,6 +3404,91 @@ private void deleteLifecyclePolicy(String volume, String bucket)
new CacheKey<>(key), CacheValue.get(1L));
}
+ /** An OBJECT_STORE bucket with S3 versioning enabled. */
+ private void createVersionedVolumeAndBucket(String volumeName,
+ String bucketName) throws IOException {
+ createVolumeAndBucket(volumeName, bucketName, OBJECT_STORE,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo versioned = metadataManager.getBucketTable().get(bucketKey)
+ .toBuilder()
+ .setVersioningStatus(BucketVersioningStatus.ENABLED)
+ .setIsVersionEnabled(true)
+ .build();
+ metadataManager.getBucketTable().put(bucketKey, versioned);
+ metadataManager.getBucketTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, versioned));
+ }
+
+ /** A key whose current version is already a delete marker. */
+ private OmKeyInfo seedDeleteMarker(String volumeName, String bucketName,
+ String keyName) throws IOException {
+ OmKeyInfo marker = OMRequestTestUtils.createOmKeyInfo(volumeName,
+ bucketName, keyName, RatisReplicationConfig.getInstance(THREE))
+ .setVersionId(1000L)
+ .setDeleteMarker(true)
+ .build();
+ metadataManager.getKeyTable(OBJECT_STORE).put(
+ metadataManager.getOzoneKey(volumeName, bucketName, keyName), marker);
+ return marker;
+ }
+
+ /** Whether the keyTable still holds the key at all. */
+ private boolean keyExists(String volumeName, String bucketName,
+ String keyName) {
+ try {
+ return metadataManager.getKeyTable(OBJECT_STORE)
+ .get(metadataManager.getOzoneKey(volumeName, bucketName, keyName))
+ != null;
+ } catch (IOException e) {
+ fail("Failed to read the key: " + e.getMessage());
+ return false;
+ }
+ }
+
+ /** A version of the key in the versionedKeyTable. */
+ private void seedNoncurrentVersion(String volumeName, String bucketName,
+ String keyName, long versionId) throws IOException {
+ OmKeyInfo version = OMRequestTestUtils.createOmKeyInfo(volumeName,
+ bucketName, keyName, RatisReplicationConfig.getInstance(THREE))
+ .setVersionId(versionId)
+ .build();
+ metadataManager.getVersionedKeyTable().put(
+ metadataManager.getVersionedOzoneKey(volumeName, bucketName, keyName,
+ versionId), version);
+ }
+
+ /** Whether the key's current version is a delete marker. */
+ private boolean isDeleteMarker(String volumeName, String bucketName,
+ String keyName) {
+ try {
+ OmKeyInfo current = metadataManager.getKeyTable(OBJECT_STORE)
+ .get(metadataManager.getOzoneKey(volumeName, bucketName, keyName));
+ return current != null && current.isDeleteMarker();
+ } catch (IOException e) {
+ fail("Failed to read the current version: " + e.getMessage());
+ return false;
+ }
+ }
+
+ /** How many versions of the key the versionedKeyTable holds. */
+ private long countNoncurrentVersions(String volumeName, String bucketName,
+ String keyName) {
+ String prefix = metadataManager.getOzoneKey(volumeName, bucketName,
+ keyName) + OzoneConsts.OM_VERSIONED_KEY_SEPARATOR;
+ long count = 0;
+ try (TableIterator>
+ iter = metadataManager.getVersionedKeyTable().iterator(prefix)) {
+ while (iter.hasNext()) {
+ iter.next();
+ count++;
+ }
+ } catch (IOException e) {
+ fail("Failed to count versions: " + e.getMessage());
+ }
+ return count;
+ }
+
private void createVolumeAndBucket(String volumeName,
String bucketName, BucketLayout bucketLayout, String owner) throws IOException {
// cheat here, just create a volume and bucket entry so that we can
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestNoncurrentVersionSelector.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestNoncurrentVersionSelector.java
new file mode 100644
index 000000000000..3120f72e2e47
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestNoncurrentVersionSelector.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.service;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLCNoncurrentVersionExpiration;
+import org.apache.hadoop.ozone.om.helpers.OmLCRule;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Tests which noncurrent versions the NoncurrentVersionExpiration rules pick.
+ *
+ * Reclaiming what was picked is OMObjectVersionsReclaimRequest's job and is
+ * tested there; this covers the choice itself.
+ */
+public class TestNoncurrentVersionSelector {
+
+ private static final String VOLUME = "vol1";
+ private static final String BUCKET = "bucket1";
+ private static final int NO_LIMIT = Integer.MAX_VALUE;
+
+ private OMMetadataManager omMetadataManager;
+ private NoncurrentVersionSelector selector;
+
+ @TempDir
+ private File folder;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath());
+ omMetadataManager = new OmMetadataManagerImpl(conf, null);
+ selector = new NoncurrentVersionSelector(omMetadataManager);
+ }
+
+ private static OmBucketInfo versionedBucket() {
+ return OmBucketInfo.newBuilder()
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setVersioningStatus(BucketVersioningStatus.ENABLED)
+ .build();
+ }
+
+ private static List rules(Integer noncurrentDays,
+ Integer newerNoncurrentVersions) throws Exception {
+ OmLCNoncurrentVersionExpiration.Builder action =
+ new OmLCNoncurrentVersionExpiration.Builder();
+ if (noncurrentDays != null) {
+ action.setNoncurrentDays(noncurrentDays);
+ }
+ if (newerNoncurrentVersions != null) {
+ action.setNewerNoncurrentVersions(newerNoncurrentVersions);
+ }
+ OmLCNoncurrentVersionExpiration expiration = action.build();
+ expiration.valid(0L);
+
+ return Collections.singletonList(new OmLCRule.Builder()
+ .setId("rule")
+ .setEnabled(true)
+ .setPrefix("")
+ .addAction(expiration)
+ .build());
+ }
+
+ private static long daysAgo(int days) {
+ return System.currentTimeMillis() - TimeUnit.DAYS.toMillis(days);
+ }
+
+ private OmKeyInfo version(String keyName, long versionId, long modTime) {
+ return new OmKeyInfo.Builder()
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setKeyName(keyName)
+ .setReplicationConfig(RatisReplicationConfig.getInstance(ONE))
+ .setVersionId(versionId)
+ .setModificationTime(modTime)
+ .setCreationTime(modTime)
+ .build();
+ }
+
+ /** A noncurrent version of a key in a bucket whose name extends BUCKET. */
+ private void addNoncurrentInSiblingBucket(String bucketName, String keyName,
+ long versionId, long noncurrentAt) throws Exception {
+ OmKeyInfo info = new OmKeyInfo.Builder()
+ .setVolumeName(VOLUME)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setReplicationConfig(RatisReplicationConfig.getInstance(ONE))
+ .setVersionId(versionId)
+ .setModificationTime(noncurrentAt)
+ .setCreationTime(noncurrentAt)
+ .setNoncurrentTime(noncurrentAt)
+ .build();
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(VOLUME, bucketName, keyName,
+ versionId),
+ info);
+ }
+
+ /** A noncurrent version stamped as having gone noncurrent at that time. */
+ private void addNoncurrent(String keyName, long versionId,
+ long noncurrentAt) throws Exception {
+ addNoncurrent(keyName, versionId, noncurrentAt, noncurrentAt);
+ }
+
+ private void addNoncurrent(String keyName, long versionId, long modTime,
+ long noncurrentAt) throws Exception {
+ addNoncurrent(keyName, versionId, modTime, noncurrentAt, 0L);
+ }
+
+ private void addNoncurrent(String keyName, long versionId, long modTime,
+ long noncurrentAt, long updateId) throws Exception {
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(VOLUME, BUCKET, keyName,
+ versionId),
+ version(keyName, versionId, modTime).toBuilder()
+ .setNoncurrentTime(noncurrentAt)
+ .setUpdateID(updateId)
+ .build());
+ }
+
+ /** A version with no stamp, as a broken invariant would leave it. */
+ private void addUnstampedNoncurrent(String keyName, long versionId,
+ long modTime) throws Exception {
+ omMetadataManager.getVersionedKeyTable().put(
+ omMetadataManager.getVersionedOzoneKey(VOLUME, BUCKET, keyName,
+ versionId),
+ version(keyName, versionId, modTime));
+ }
+
+ private void addCurrent(String keyName, long versionId, long modTime)
+ throws Exception {
+ omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE).put(
+ omMetadataManager.getOzoneKey(VOLUME, BUCKET, keyName),
+ version(keyName, versionId, modTime));
+ }
+
+ /** The dbKeys of what a pass selected, in the order it met them. */
+ private static List keysOf(
+ List records) {
+ return records.stream()
+ .map(NoncurrentVersionSelector.ExpiredRecord::getDbKey)
+ .collect(Collectors.toList());
+ }
+
+ private String versionKey(String keyName, long versionId) {
+ return omMetadataManager.getVersionedOzoneKey(VOLUME, BUCKET, keyName,
+ versionId);
+ }
+
+ // ---- NewerNoncurrentVersions ----
+
+ /**
+ * A dbKey names a position rather than a record, so what is selected carries
+ * the updateID of the record the rules matched. The reclaim runs after
+ * replication, by which time the key may hold something else.
+ */
+ @Test
+ public void testWhatIsSelectedCarriesTheRecordsUpdateId() throws Exception {
+ addCurrent("key", 500L, daysAgo(0));
+ addNoncurrent("key", 300L, daysAgo(1), daysAgo(1), 4711L);
+ addNoncurrent("key", 200L, daysAgo(2), daysAgo(2), 4712L);
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 1), null, NO_LIMIT);
+
+ assertEquals(Collections.singletonList(new
+ NoncurrentVersionSelector.ExpiredRecord(versionKey("key", 200L),
+ 4712L)), selection.getExpiredVersions());
+ }
+
+ /** Versions are visited newest first, so the ones over the limit are oldest. */
+ @Test
+ public void testTheOldestVersionsOverTheLimitAreSelected() throws Exception {
+ addCurrent("key", 500L, daysAgo(0));
+ addNoncurrent("key", 400L, daysAgo(1));
+ addNoncurrent("key", 300L, daysAgo(2));
+ addNoncurrent("key", 200L, daysAgo(3));
+ addNoncurrent("key", 100L, daysAgo(4));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 2), null, NO_LIMIT);
+
+ assertEquals(Arrays.asList(versionKey("key", 200L),
+ versionKey("key", 100L)), keysOf(selection.getExpiredVersions()));
+ assertTrue(selection.isFinished());
+ }
+
+ /** The count restarts at every key, so one key cannot spend another's. */
+ @Test
+ public void testEachKeyIsCountedSeparately() throws Exception {
+ for (String keyName : Arrays.asList("a", "b", "c")) {
+ addCurrent(keyName, 500L, daysAgo(0));
+ addNoncurrent(keyName, 300L, daysAgo(1));
+ addNoncurrent(keyName, 200L, daysAgo(2));
+ }
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 1), null, NO_LIMIT);
+
+ // one over the limit per key, never more
+ assertEquals(Arrays.asList(versionKey("a", 200L), versionKey("b", 200L),
+ versionKey("c", 200L)), keysOf(selection.getExpiredVersions()));
+ }
+
+ @Test
+ public void testAKeyWithinTheLimitIsUntouched() throws Exception {
+ addCurrent("key", 500L, daysAgo(0));
+ addNoncurrent("key", 300L, daysAgo(1));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 5), null, NO_LIMIT);
+
+ assertTrue(selection.getExpiredVersions().isEmpty());
+ }
+
+ // ---- NoncurrentDays ----
+
+ /**
+ * A version's age is counted from when it stopped being current, which the
+ * record carries, and not from its own modification time.
+ */
+ @Test
+ public void testAgeIsCountedFromWhenTheVersionWasSuperseded()
+ throws Exception {
+ addCurrent("key", 500L, daysAgo(1));
+ // written long ago, but only superseded yesterday
+ addNoncurrent("key", 400L, daysAgo(90), daysAgo(1));
+ // written at the same time, and noncurrent ever since
+ addNoncurrent("key", 300L, daysAgo(90), daysAgo(90));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(30, null), null, NO_LIMIT);
+
+ assertEquals(Collections.singletonList(versionKey("key", 300L)),
+ keysOf(selection.getExpiredVersions()),
+ "a version superseded yesterday is not 30 days noncurrent");
+ }
+
+ /**
+ * The moment a version went noncurrent is fixed when it happens, so
+ * removing the version that superseded it does not restart its clock. A
+ * derived age would: with the version above gone, the current version is
+ * the only thing left to date it against.
+ */
+ @Test
+ public void testRemovingTheSupersedingVersionKeepsTheClock()
+ throws Exception {
+ addCurrent("key", 500L, daysAgo(1));
+ // 400 superseded it 90 days ago and has since been reclaimed itself
+ addNoncurrent("key", 300L, daysAgo(120), daysAgo(90));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(30, null), null, NO_LIMIT);
+
+ assertEquals(Collections.singletonList(versionKey("key", 300L)),
+ keysOf(selection.getExpiredVersions()));
+ }
+
+ /**
+ * Every record is stamped as it is moved into the versionedKeyTable, so an
+ * unstamped one means that invariant is broken, and expiring on a broken
+ * invariant would destroy data.
+ */
+ @Test
+ public void testAnUnstampedVersionIsNotExpiredByAge() throws Exception {
+ addCurrent("key", 500L, daysAgo(1));
+ addUnstampedNoncurrent("key", 400L, daysAgo(90));
+ addUnstampedNoncurrent("key", 300L, daysAgo(120));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(30, null), null, NO_LIMIT);
+
+ assertTrue(selection.getExpiredVersions().isEmpty());
+
+ // the count rule does not depend on the stamp, so it still applies
+ selection = selector.select(versionedBucket(), rules(null, 1), null,
+ NO_LIMIT);
+
+ assertEquals(Collections.singletonList(versionKey("key", 300L)),
+ keysOf(selection.getExpiredVersions()));
+ }
+
+ /** Either limit on its own is enough, so each catches what the other misses. */
+ @Test
+ public void testCountAndAgeCombine() throws Exception {
+ // 400 went noncurrent a day ago, 300 went noncurrent 100 days ago
+ addCurrent("key", 500L, daysAgo(1));
+ addNoncurrent("key", 400L, daysAgo(1));
+ addNoncurrent("key", 300L, daysAgo(100));
+
+ // age catches 300 even though the count limit is nowhere near
+ NoncurrentVersionSelector.Selection byAge =
+ selector.select(versionedBucket(), rules(30, 5), null, NO_LIMIT);
+ assertEquals(Collections.singletonList(versionKey("key", 300L)),
+ keysOf(byAge.getExpiredVersions()));
+
+ // count catches 300 even though it is nowhere near the age limit
+ NoncurrentVersionSelector.Selection byCount =
+ selector.select(versionedBucket(), rules(3650, 1), null, NO_LIMIT);
+ assertEquals(Collections.singletonList(versionKey("key", 300L)),
+ keysOf(byCount.getExpiredVersions()));
+ }
+
+ // ---- resume ----
+
+ /**
+ * A pass stops at a key boundary and reports where to pick up, so the next
+ * pass counts that key's versions from the start rather than mid-key.
+ */
+ @Test
+ public void testAPassResumesAtAKeyBoundary() throws Exception {
+ for (String keyName : Arrays.asList("a", "b", "c")) {
+ addCurrent(keyName, 500L, daysAgo(0));
+ addNoncurrent(keyName, 300L, daysAgo(1));
+ addNoncurrent(keyName, 200L, daysAgo(2));
+ }
+
+ NoncurrentVersionSelector.Selection first =
+ selector.select(versionedBucket(), rules(null, 1), null, 1);
+
+ assertEquals(Collections.singletonList(versionKey("a", 200L)),
+ keysOf(first.getExpiredVersions()));
+ assertFalse(first.isFinished());
+ // the boundary is b's newest version, never the middle of a key
+ assertEquals(versionKey("b", 300L), first.getResumeFrom());
+
+ NoncurrentVersionSelector.Selection second = selector.select(
+ versionedBucket(), rules(null, 1), first.getResumeFrom(), NO_LIMIT);
+
+ assertEquals(Arrays.asList(versionKey("b", 200L), versionKey("c", 200L)),
+ keysOf(second.getExpiredVersions()));
+ assertTrue(second.isFinished());
+ assertNull(second.getResumeFrom());
+ }
+
+ @Test
+ public void testAPassStopsAfterWalkingTheLimitEvenIfNothingExpires()
+ throws Exception {
+ // Nothing here expires: every key keeps two noncurrent versions and the
+ // rule keeps five. Bounding the pass by what it selected would walk the
+ // whole bucket in one go, never yielding and never saving progress.
+ for (String keyName : Arrays.asList("a", "b", "c")) {
+ addCurrent(keyName, 500L, daysAgo(0));
+ addNoncurrent(keyName, 300L, daysAgo(1));
+ addNoncurrent(keyName, 200L, daysAgo(2));
+ }
+
+ NoncurrentVersionSelector.Selection first =
+ selector.select(versionedBucket(), rules(null, 5), null, 2);
+
+ assertTrue(first.getExpiredVersions().isEmpty());
+ assertFalse(first.isFinished());
+ assertEquals(2, first.getVersionsScanned());
+ assertEquals(versionKey("b", 300L), first.getResumeFrom());
+
+ NoncurrentVersionSelector.Selection second = selector.select(
+ versionedBucket(), rules(null, 5), first.getResumeFrom(), 2);
+
+ assertEquals(2, second.getVersionsScanned());
+ assertEquals(versionKey("c", 300L), second.getResumeFrom());
+
+ NoncurrentVersionSelector.Selection third = selector.select(
+ versionedBucket(), rules(null, 5), second.getResumeFrom(), 2);
+
+ assertEquals(2, third.getVersionsScanned());
+ assertTrue(third.isFinished());
+ }
+
+ @Test
+ public void testTheScannedCountCoversSurvivorsAndExpiredAlike()
+ throws Exception {
+ addCurrent("a", 500L, daysAgo(0));
+ addNoncurrent("a", 300L, daysAgo(1));
+ addNoncurrent("a", 200L, daysAgo(2));
+ addNoncurrent("a", 100L, daysAgo(3));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 1), null, NO_LIMIT);
+
+ assertEquals(2, selection.getExpiredVersions().size());
+ assertEquals(3, selection.getVersionsScanned());
+ }
+
+ @Test
+ public void testAnEmptyBucketFinishesImmediately() throws Exception {
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 1), null, NO_LIMIT);
+
+ assertTrue(selection.getExpiredVersions().isEmpty());
+ assertTrue(selection.isFinished());
+ assertEquals(0, selection.getVersionsScanned());
+ }
+
+ /**
+ * The scan is bounded by the bucket. Without the separator its prefix is
+ * also a prefix of every bucket whose name extends it, and this bucket's
+ * rules would expire another bucket's versions.
+ */
+ @Test
+ public void testABucketWhoseNameExtendsThisOneIsNotScanned()
+ throws Exception {
+ addNoncurrent("key", 300L, daysAgo(1));
+ addNoncurrent("key", 200L, daysAgo(2));
+ addNoncurrentInSiblingBucket(BUCKET + "0", "key", 300L, daysAgo(1));
+ addNoncurrentInSiblingBucket(BUCKET + "0", "key", 200L, daysAgo(2));
+
+ NoncurrentVersionSelector.Selection selection =
+ selector.select(versionedBucket(), rules(null, 1), null, NO_LIMIT);
+
+ assertEquals(2, selection.getVersionsScanned());
+ assertEquals(Collections.singletonList(versionKey("key", 200L)),
+ keysOf(selection.getExpiredVersions()));
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
index ec9ada89a9f1..1786b22ae754 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.om.helpers.OmLCAbortIncompleteMultipartUpload;
import org.apache.hadoop.ozone.om.helpers.OmLCExpiration;
import org.apache.hadoop.ozone.om.helpers.OmLCFilter;
+import org.apache.hadoop.ozone.om.helpers.OmLCNoncurrentVersionExpiration;
import org.apache.hadoop.ozone.om.helpers.OmLCRule;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleRuleAndOperator;
@@ -80,6 +81,9 @@ public static class Rule {
@XmlElement(name = "AbortIncompleteMultipartUpload")
private AbortIncompleteMultipartUpload abortIncompleteMultipartUpload;
+ @XmlElement(name = "NoncurrentVersionExpiration")
+ private NoncurrentVersionExpiration noncurrentVersionExpiration;
+
@XmlElement(name = "Filter")
private Filter filter;
@@ -119,6 +123,15 @@ public AbortIncompleteMultipartUpload getAbortIncompleteMultipartUpload() {
return abortIncompleteMultipartUpload;
}
+ public NoncurrentVersionExpiration getNoncurrentVersionExpiration() {
+ return noncurrentVersionExpiration;
+ }
+
+ public void setNoncurrentVersionExpiration(
+ NoncurrentVersionExpiration noncurrentVersionExpiration) {
+ this.noncurrentVersionExpiration = noncurrentVersionExpiration;
+ }
+
public void setAbortIncompleteMultipartUpload(AbortIncompleteMultipartUpload abortIncompleteMultipartUpload) {
this.abortIncompleteMultipartUpload = abortIncompleteMultipartUpload;
}
@@ -144,6 +157,17 @@ public static class Expiration {
@XmlElement(name = "Date")
private String date;
+ @XmlElement(name = "ExpiredObjectDeleteMarker")
+ private Boolean expiredObjectDeleteMarker;
+
+ public Boolean getExpiredObjectDeleteMarker() {
+ return expiredObjectDeleteMarker;
+ }
+
+ public void setExpiredObjectDeleteMarker(Boolean removeMarker) {
+ this.expiredObjectDeleteMarker = removeMarker;
+ }
+
public Integer getDays() {
return days;
}
@@ -161,6 +185,36 @@ public void setDate(String date) {
}
}
+ /**
+ * NoncurrentVersionExpiration entity for lifecycle rule: reclaims the
+ * noncurrent versions of a key on a bucket with versioning enabled.
+ */
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlRootElement(name = "NoncurrentVersionExpiration")
+ public static class NoncurrentVersionExpiration {
+ @XmlElement(name = "NoncurrentDays")
+ private Integer noncurrentDays;
+
+ @XmlElement(name = "NewerNoncurrentVersions")
+ private Integer newerNoncurrentVersions;
+
+ public Integer getNoncurrentDays() {
+ return noncurrentDays;
+ }
+
+ public void setNoncurrentDays(Integer noncurrentDays) {
+ this.noncurrentDays = noncurrentDays;
+ }
+
+ public Integer getNewerNoncurrentVersions() {
+ return newerNoncurrentVersions;
+ }
+
+ public void setNewerNoncurrentVersions(Integer newerNoncurrentVersions) {
+ this.newerNoncurrentVersions = newerNoncurrentVersions;
+ }
+ }
+
/**
* AbortIncompleteMultipartUpload entity for lifecycle rule.
*/
@@ -336,6 +390,10 @@ private OmLCRule convertToOmRule(Rule rule) throws OMException, OS3Exception {
if (rule.getAbortIncompleteMultipartUpload() != null) {
builder.addAction(convertToOmAbortIncompleteMultipartUpload(rule.getAbortIncompleteMultipartUpload()));
}
+ if (rule.getNoncurrentVersionExpiration() != null) {
+ builder.addAction(convertToOmNoncurrentVersionExpiration(
+ rule.getNoncurrentVersionExpiration()));
+ }
if (rule.getFilter() != null) {
builder.setFilter(convertToOmFilter(rule.getFilter()));
}
@@ -358,6 +416,30 @@ private OmLCExpiration convertToOmExpiration(Expiration expiration) throws OMExc
if (expiration.getDate() != null) {
builder.setDate(expiration.getDate());
}
+ if (expiration.getExpiredObjectDeleteMarker() != null) {
+ builder.setExpiredObjectDeleteMarker(
+ expiration.getExpiredObjectDeleteMarker());
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Converts S3 NoncurrentVersionExpiration to internal representation.
+ */
+ private OmLCNoncurrentVersionExpiration convertToOmNoncurrentVersionExpiration(
+ NoncurrentVersionExpiration noncurrentVersionExpiration) {
+ OmLCNoncurrentVersionExpiration.Builder builder =
+ new OmLCNoncurrentVersionExpiration.Builder();
+
+ if (noncurrentVersionExpiration.getNoncurrentDays() != null) {
+ builder.setNoncurrentDays(
+ noncurrentVersionExpiration.getNoncurrentDays());
+ }
+ if (noncurrentVersionExpiration.getNewerNoncurrentVersions() != null) {
+ builder.setNewerNoncurrentVersions(
+ noncurrentVersionExpiration.getNewerNoncurrentVersions());
+ }
return builder.build();
}
@@ -465,6 +547,11 @@ private static Rule convertFromOzoneRule(OzoneLifecycleConfiguration.OzoneLCRule
rule.setAbortIncompleteMultipartUpload(
convertFromOzoneAbortIncompleteMultipartUpload(ozoneRule.getAbortIncompleteMultipartUpload()));
}
+ if (ozoneRule.getNoncurrentVersionExpiration() != null) {
+ rule.setNoncurrentVersionExpiration(
+ convertFromOzoneNoncurrentVersionExpiration(
+ ozoneRule.getNoncurrentVersionExpiration()));
+ }
if (ozoneRule.getFilter() != null) {
rule.setFilter(convertFromOzoneFilter(ozoneRule.getFilter()));
}
@@ -472,6 +559,27 @@ private static Rule convertFromOzoneRule(OzoneLifecycleConfiguration.OzoneLCRule
return rule;
}
+ /**
+ * Converts an Ozone internal NoncurrentVersionExpiration to S3.
+ */
+ private static NoncurrentVersionExpiration
+ convertFromOzoneNoncurrentVersionExpiration(
+ OzoneLifecycleConfiguration.OzoneLCNoncurrentVersionExpiration ozone) {
+
+ NoncurrentVersionExpiration expiration = new NoncurrentVersionExpiration();
+
+ if (ozone.getNoncurrentDays() != null && ozone.getNoncurrentDays() > 0) {
+ expiration.setNoncurrentDays(ozone.getNoncurrentDays());
+ }
+ if (ozone.getNewerNoncurrentVersions() != null
+ && ozone.getNewerNoncurrentVersions() > 0) {
+ expiration.setNewerNoncurrentVersions(
+ ozone.getNewerNoncurrentVersions());
+ }
+
+ return expiration;
+ }
+
/**
* Converts an Ozone internal expiration to S3 expiration.
*
@@ -490,6 +598,9 @@ private static Expiration convertFromOzoneExpiration(
if (ozoneExpiration.getDays() != null && ozoneExpiration.getDays() > 0) {
expiration.setDays(ozoneExpiration.getDays());
}
+ if (ozoneExpiration.isExpiredObjectDeleteMarker()) {
+ expiration.setExpiredObjectDeleteMarker(true);
+ }
return expiration;
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 09ae82564a5a..85e0acd7bd38 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -888,9 +888,17 @@ private static OzoneLifecycleConfiguration toOzoneLifecycleConfiguration(
OzoneLifecycleConfiguration.OzoneLCAbortIncompleteMultipartUpload a = null;
OzoneLifecycleConfiguration.OzoneLCFilter f = null;
+ OzoneLifecycleConfiguration.OzoneLCNoncurrentVersionExpiration n = null;
+
if (r.getExpiration() != null) {
e = new OzoneLifecycleConfiguration.OzoneLCExpiration(
- r.getExpiration().getDays(), r.getExpiration().getDate());
+ r.getExpiration().getDays(), r.getExpiration().getDate(),
+ r.getExpiration().isExpiredObjectDeleteMarker());
+ }
+ if (r.getNoncurrentVersionExpiration() != null) {
+ n = new OzoneLifecycleConfiguration.OzoneLCNoncurrentVersionExpiration(
+ r.getNoncurrentVersionExpiration().getNoncurrentDays(),
+ r.getNoncurrentVersionExpiration().getNewerNoncurrentVersions());
}
if (r.getAbortIncompleteMultipartUpload() != null) {
a = new OzoneLifecycleConfiguration.OzoneLCAbortIncompleteMultipartUpload(
@@ -907,7 +915,7 @@ private static OzoneLifecycleConfiguration toOzoneLifecycleConfiguration(
}
rules.add(new OzoneLifecycleConfiguration.OzoneLCRule(r.getId(),
- r.getEffectivePrefix(), (r.isEnabled() ? "Enabled" : "Disabled"), e, a, f));
+ r.getEffectivePrefix(), (r.isEnabled() ? "Enabled" : "Disabled"), e, a, n, f));
}
return new OzoneLifecycleConfiguration(
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
index 5080f192b9c4..1d21038eef40 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
@@ -21,6 +21,7 @@
import static java.net.HttpURLConnection.HTTP_NO_CONTENT;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_LIFECYCLE_CONFIGURATION;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.ByteArrayInputStream;
@@ -80,6 +81,48 @@ public void testDeleteLifecycleConfiguration() throws Exception {
}
}
+ /**
+ * A configuration carrying the versioning actions has to be removable like
+ * any other: the rule model round-trips through Put, Get and Delete.
+ */
+ @Test
+ public void testDeleteLifecycleConfigurationWithVersioningActions()
+ throws Exception {
+ String bucketName = "bucket1";
+ bucketEndpoint.put(bucketName, versioningRules());
+ Response r = bucketEndpoint.delete(bucketName);
+
+ assertEquals(HTTP_NO_CONTENT, r.getStatus());
+
+ OS3Exception ex = assertThrows(OS3Exception.class,
+ () -> bucketEndpoint.get(bucketName));
+ assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
+ assertEquals(NO_SUCH_LIFECYCLE_CONFIGURATION.getCode(), ex.getCode());
+ }
+
+ private static InputStream versioningRules() {
+ String xml = ""
+ + ""
+ + "trim noncurrent versions"
+ + "prefix/"
+ + "Enabled"
+ + ""
+ + "30"
+ + ""
+ + ""
+ + ""
+ + "sweep expired delete markers"
+ + "prefix/"
+ + "Enabled"
+ + ""
+ + "true"
+ + ""
+ + ""
+ + "";
+ return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
+ }
+
private static InputStream getBody() {
String xml = ("" +
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
index 8e4d0eb88ada..0f8c67ee2701 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
@@ -84,6 +84,56 @@ public void testGetLifecycleConfiguration() throws Exception {
lcc.getRules().get(0).getExpiration().getDays().intValue());
}
+ /**
+ * The versioning actions have to survive Put followed by Get: the rule the
+ * gateway parsed is the rule an operator gets back.
+ */
+ @Test
+ public void testGetLifecycleWithVersioningActions() throws Exception {
+ String bucketName = "bucket1";
+ bucketEndpoint.put(bucketName, versioningRules());
+ Response r = bucketEndpoint.get(bucketName);
+
+ assertEquals(HTTP_OK, r.getStatus());
+ S3LifecycleConfiguration lcc = (S3LifecycleConfiguration) r.getEntity();
+ assertEquals(2, lcc.getRules().size());
+
+ S3LifecycleConfiguration.Rule noncurrent = lcc.getRules().get(0);
+ assertEquals(30, noncurrent.getNoncurrentVersionExpiration()
+ .getNoncurrentDays().intValue());
+ assertEquals(5, noncurrent.getNoncurrentVersionExpiration()
+ .getNewerNoncurrentVersions().intValue());
+
+ S3LifecycleConfiguration.Rule marker = lcc.getRules().get(1);
+ assertEquals(Boolean.TRUE,
+ marker.getExpiration().getExpiredObjectDeleteMarker());
+ assertNull(marker.getExpiration().getDays());
+ }
+
+ private static InputStream versioningRules() {
+ String xml = ""
+ + ""
+ + "trim noncurrent versions"
+ + "prefix/"
+ + "Enabled"
+ + ""
+ + "30"
+ + "5"
+ + ""
+ + ""
+ + ""
+ + "sweep expired delete markers"
+ + "prefix/"
+ + "Enabled"
+ + ""
+ + "true"
+ + ""
+ + ""
+ + "";
+ return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
+ }
+
@Test
public void testGetLifecycleWithAbortIncompleteMultipartUpload() throws Exception {
String bucketName = "bucket1";
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
index d76ff51b66bb..9534f8abdb57 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
@@ -191,6 +191,64 @@ public void testPutValidLifecycleConfiguration() throws Exception {
assertEquals(HTTP_OK, bucketEndpoint.put("bucket1", onePrefixTwoTags()).getStatus());
}
+ @Test
+ public void testPutInvalidVersioningLifecycleConfiguration()
+ throws Exception {
+ // NoncurrentVersionExpiration that limits neither age nor count
+ testInvalidLifecycleConfiguration(
+ TestS3LifecycleConfigurationPut::withEmptyNoncurrentExpiration,
+ HTTP_BAD_REQUEST, INVALID_REQUEST.getCode());
+ // ExpiredObjectDeleteMarker combined with an age
+ testInvalidLifecycleConfiguration(
+ TestS3LifecycleConfigurationPut::withMarkerAndDays,
+ HTTP_BAD_REQUEST, INVALID_REQUEST.getCode());
+ // NewerNoncurrentVersions above S3's ceiling
+ testInvalidLifecycleConfiguration(
+ TestS3LifecycleConfigurationPut::withTooManyNoncurrentVersions,
+ HTTP_BAD_REQUEST, INVALID_REQUEST.getCode());
+ }
+
+ private static InputStream withEmptyNoncurrentExpiration() {
+ return xml(
+ "" +
+ "limits nothing" +
+ "prefix/" +
+ "Enabled" +
+ "" +
+ "");
+ }
+
+ private static InputStream withMarkerAndDays() {
+ return xml(
+ "" +
+ "marker and days" +
+ "prefix/" +
+ "Enabled" +
+ "" +
+ "30" +
+ "true" +
+ "" +
+ "");
+ }
+
+ private static InputStream withTooManyNoncurrentVersions() {
+ return xml(
+ "" +
+ "too many" +
+ "prefix/" +
+ "Enabled" +
+ "" +
+ "101" +
+ "" +
+ "");
+ }
+
+ private static InputStream xml(String rules) {
+ String doc = "" + rules + "";
+ return new ByteArrayInputStream(doc.getBytes(StandardCharsets.UTF_8));
+ }
+
@Test
public void testPutLifecycleConfigurationFailsWithNonBucketOwner()
throws Exception {