diff --git a/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs b/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs index 7c12e72e..1050ceb7 100644 --- a/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs +++ b/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs @@ -1,5 +1,5 @@ /* - * Copyright 2025, Optimizely + * Copyright 2025, 2026, Optimizely * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -411,18 +411,13 @@ public void TestLocalHoldouts_GlobalHoldoutEvaluatedBeforePerRuleLogic() } [Test] - public void TestLocalHoldouts_UserBucketedIntoLocalHoldoutForDeliveryRuleReturnsHoldoutVariation() + public void TestLocalHoldouts_DeliveryRuleHoldoutHitSkipsRuleAndLaterRuleServes() { - // When a user hits a local holdout targeting delivery rule X, the holdout variation - // is returned and the rule's own audience/traffic checks are not evaluated. - // holdout_local_rule1 targets rule_id_1 with 100% traffic. - // To test this without global holdout interference, we use a config where - // global holdout has 0% traffic — we manipulate the global holdout in-place. InitializeLocalHoldoutsConfig(); - // Remove global holdout traffic so it doesn't intercept - var globalHoldout = LocalHoldoutsConfig.GetGlobalHoldouts()[0]; - globalHoldout.TrafficAllocation = new TrafficAllocation[0]; // 0% traffic + LocalHoldoutsConfig.GetGlobalHoldouts()[0].TrafficAllocation = new TrafficAllocation[0]; + LocalHoldoutsConfig.GetHoldoutsForRule("rule_id_2")[0].TrafficAllocation = + new TrafficAllocation[0]; var realBucketer = new Bucketer(LoggerMock.Object); var decisionService = new DecisionService(realBucketer, @@ -444,12 +439,15 @@ public void TestLocalHoldouts_UserBucketedIntoLocalHoldoutForDeliveryRuleReturns var decision = result[0].ResultObject; Assert.IsNotNull(decision, "Decision should not be null"); - Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.Source, - "Decision source should be holdout for local holdout hit"); - Assert.AreEqual("local_holdout_rule1", decision.Experiment?.Key, - "Decision should be from local holdout targeting rule_id_1"); - Assert.AreEqual("local_holdout_off", decision.Variation?.Key, - "Variation should be the holdout's variation"); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_ROLLOUT, decision.Source); + Assert.AreEqual("rule_id_2", decision.Experiment?.Id, + "Held-out rule_id_1 should be skipped and rule_id_2 should serve"); + Assert.AreEqual("rule2_var_1", decision.Variation?.Id); + Assert.AreEqual(1, decision.HoldoutDecisions.Count); + Assert.AreEqual("local_holdout_rule1", decision.HoldoutDecisions[0].Experiment.Key); + Assert.AreEqual("local_holdout_off", decision.HoldoutDecisions[0].Variation.Key); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, + decision.HoldoutDecisions[0].Source); } [Test] @@ -595,6 +593,7 @@ public void TestForcedDecisionBeats100PercentLocalHoldout() "Forced decision must NOT return holdout source — forced decision takes priority over local holdout"); Assert.AreEqual("variation_a", decision.Variation?.Key, "Forced decision variation should be returned, not holdout variation"); + Assert.AreEqual(0, decision.HoldoutDecisions.Count); } [Test] @@ -639,10 +638,12 @@ public void TestLocalHoldouts_AppliesToExperimentRules() var decision = result[0].ResultObject; Assert.IsNotNull(decision, "Decision should not be null"); - Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.Source, - "Decision source should be holdout since local holdout targets the experiment rule"); - Assert.AreEqual("local_holdout_exp_rule1", decision.Experiment?.Key, - "Decision experiment should be the local holdout targeting exp_rule_id_1"); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_FEATURE_TEST, decision.Source); + Assert.AreEqual("experiment_rule_2", decision.Experiment?.Key, + "Held out of experiment_rule_1, user should be served by experiment_rule_2"); + Assert.AreEqual("variation_b", decision.Variation?.Key); + Assert.AreEqual(1, decision.HoldoutDecisions.Count); + Assert.AreEqual("local_holdout_exp_rule1", decision.HoldoutDecisions[0].Experiment.Key); } // ===================================================================== @@ -784,8 +785,8 @@ public void TestExcludeTargetedDeliveries_True_ABRuleStillBlocked() "With excludeTargetedDeliveries=true, A/B experiment rules should be skipped (not evaluated)"); Assert.IsNull(decision.Variation, "Variation should be null since A/B was blocked and no TD rule matched"); - Assert.IsNotNull(decision.HoldoutDecision, - "HoldoutDecision should be attached showing the user was in a holdout that blocked A/B rules"); + Assert.AreEqual(1, decision.HoldoutDecisions.Count, + "HoldoutDecisions should be attached showing the user was in a holdout that blocked A/B rules"); } [Test] @@ -829,8 +830,11 @@ public void TestLocalHoldout_ExcludeTargetedDeliveries_True_StillAppliesForTDRul var decision = result[0].ResultObject; Assert.IsNotNull(decision); - Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.Source, - "Local holdout must apply to TD rules even when excludeTargetedDeliveries=true (flag is only for global holdouts)"); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_ROLLOUT, decision.Source); + Assert.AreEqual("rule_id_everyone_else", decision.Experiment?.Id); + CollectionAssert.AreEqual(new[] { "local_holdout_rule1", "local_holdout_rule2" }, + decision.HoldoutDecisions.Select(h => h.Experiment.Key).ToArray(), + "Local holdouts must apply to TD rules even when excludeTargetedDeliveries=true (flag is only for global holdouts)"); } [Test] @@ -858,7 +862,9 @@ public void TestLocalHoldout_ExcludeTargetedDeliveries_True_StillAppliesForABRul var decision = result[0].ResultObject; Assert.IsNotNull(decision); - Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.Source, + Assert.AreEqual("experiment_rule_2", decision.Experiment?.Key); + CollectionAssert.AreEqual(new[] { "local_holdout_exp_rule1" }, + decision.HoldoutDecisions.Select(h => h.Experiment.Key).ToArray(), "Local holdout with excludeTargetedDeliveries=true should still apply for A/B rules"); } @@ -903,8 +909,8 @@ public void TestGlobalHoldout_ExcludeTargetedDeliveries_NoTDRuleMatch_ReturnsNul "When TD returns no match with excludeTargetedDeliveries=true, variation should be null (no holdout fallback)"); Assert.IsNull(decision.Experiment, "When TD returns no match with excludeTargetedDeliveries=true, experiment should be null (no holdout fallback)"); - Assert.IsNotNull(decision.HoldoutDecision, - "HoldoutDecision should be attached for event dispatch even when decision is null"); + Assert.AreEqual(1, decision.HoldoutDecisions.Count, + "HoldoutDecisions should be attached for event dispatch even when decision is null"); } [Test] public void TestExcludeTargetedDeliveries_HoldoutEventSentWhenTDBypasses() @@ -988,10 +994,10 @@ public void TestExcludeTargetedDeliveries_HoldoutDecisionAttachedToTDResult() Assert.IsNotNull(decision); Assert.AreEqual(FeatureDecision.DECISION_SOURCE_ROLLOUT, decision.Source, "TD rule should be returned as the decision"); - Assert.IsNotNull(decision.HoldoutDecision, - "HoldoutDecision should be attached when holdout was bypassed for TD"); - Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.HoldoutDecision.Source, - "Attached HoldoutDecision should have holdout source"); + Assert.AreEqual(1, decision.HoldoutDecisions.Count, + "HoldoutDecisions should be attached when holdout was bypassed for TD"); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_HOLDOUT, decision.HoldoutDecisions[0].Source, + "Attached holdout decision should have holdout source"); } [Test] @@ -1055,6 +1061,261 @@ public void TestExcludeTargetedDeliveries_HoldoutEventSentWhenNoTDMatch_Dispatch "decisionEventDispatched should be true when the holdout impression is dispatched, " + "even if the primary (TD) decision is null"); } + + private Optimizely CreateLocalHoldoutsOptimizelyWithMockedEvents( + Action mutateDatafile = null + ) + { + var datafile = (JObject)TestData["datafileWithLocalHoldouts"].DeepClone(); + datafile["holdouts"][0]["trafficAllocation"] = new JArray(); + mutateDatafile?.Invoke(datafile); + + EventProcessorMock.Setup(ep => ep.Process(It.IsAny())); + return new Optimizely(datafile.ToString(), null, LoggerMock.Object, + new NoOpErrorHandler(), null, false, EventProcessorMock.Object); + } + + private static void ZeroTraffic(JObject datafile, string section, string id) + { + var entity = datafile[section].First(e => (string)e["id"] == id); + entity["trafficAllocation"] = new JArray(); + } + + private void VerifyHoldoutImpression(string holdoutKey, Times times) + { + EventProcessorMock.Verify(ep => ep.Process(It.Is(ie => + ie.Experiment.Key == holdoutKey && + ie.Metadata.RuleType == FeatureDecision.DECISION_SOURCE_HOLDOUT + )), times); + } + + [Test] + public void TestLocalHoldouts_HitsOnSeveralRulesSendSeveralHoldoutEvents() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_1"); + + Assert.AreEqual("everyone_else_rule", decision.RuleKey); + Assert.IsTrue(decision.Enabled); + VerifyHoldoutImpression("local_holdout_rule1", Times.Once()); + VerifyHoldoutImpression("local_holdout_rule2", Times.Once()); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestLocalHoldouts_HoldoutHitThenNoRuleServes() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(df => + { + foreach (var rule in df["rollouts"][0]["experiments"]) + { + rule["trafficAllocation"] = new JArray(); + } + }); + var config = optimizely.ProjectConfigManager.GetConfig() as DatafileProjectConfig; + var decisionService = new DecisionService(new Bucketer(LoggerMock.Object), + new NoOpErrorHandler(), null, LoggerMock.Object, null); + var userContext = optimizely.CreateUserContext(TestUserId); + + var result = decisionService.GetVariationsForFeatureList( + new List { config.FeatureKeyMap["test_flag_1"] }, userContext, config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + var featureDecision = result[0].ResultObject; + Assert.IsNull(featureDecision.Variation); + Assert.IsNull(featureDecision.Experiment); + Assert.AreEqual(FeatureDecision.DECISION_SOURCE_ROLLOUT, featureDecision.Source); + Assert.AreEqual(2, featureDecision.HoldoutDecisions.Count); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsFalse(decision.Enabled); + Assert.IsNull(decision.VariationKey); + Assert.IsNull(decision.RuleKey); + VerifyHoldoutImpression("local_holdout_rule1", Times.Once()); + VerifyHoldoutImpression("local_holdout_rule2", Times.Once()); + } + + [Test] + public void TestLocalHoldouts_DeliveryRuleHitWithSendFlagDecisionsOffStillSendsHoldoutEvent() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(df => + { + df["sendFlagDecisions"] = false; + ZeroTraffic(df, "localHoldouts", "holdout_local_rule2"); + }); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_1"); + + Assert.AreEqual("delivery_rule_2", decision.RuleKey); + VerifyHoldoutImpression("local_holdout_rule1", Times.Once()); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Once(), + "Only the holdout event should be sent; the rollout event is suppressed by sendFlagDecisions=false"); + } + + [Test] + public void TestLocalHoldouts_DisableDecisionEventSuppressesHoldoutEvents() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_1", + new[] { OptimizelyDecideOption.DISABLE_DECISION_EVENT }); + + Assert.AreEqual("everyone_else_rule", decision.RuleKey); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Never()); + } + + [Test] + public void TestLocalHoldouts_HeldOutOfFirstExperimentRuleServedBySecond() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_2"); + + Assert.AreEqual("experiment_rule_2", decision.RuleKey); + Assert.AreEqual("variation_b", decision.VariationKey); + VerifyHoldoutImpression("local_holdout_exp_rule1", Times.Once()); + EventProcessorMock.Verify(ep => ep.Process(It.Is(ie => + ie.Experiment.Key == "experiment_rule_2" && + ie.Metadata.RuleType == FeatureDecision.DECISION_SOURCE_FEATURE_TEST + )), Times.Once()); + } + + [Test] + public void TestLocalHoldouts_OneHoldoutTargetingTwoRulesSendsTwoHoldoutEvents() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(df => + { + var holdouts = (JArray)df["localHoldouts"]; + holdouts.First(h => (string)h["id"] == "holdout_local_rule1")["includedRules"] = + new JArray("rule_id_1", "rule_id_2"); + holdouts.First(h => (string)h["id"] == "holdout_local_rule2")["includedRules"] = + new JArray(); + }); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_1"); + + Assert.AreEqual("everyone_else_rule", decision.RuleKey); + VerifyHoldoutImpression("local_holdout_rule1", Times.Exactly(2)); + VerifyHoldoutImpression("local_holdout_rule2", Times.Never()); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestLocalHoldouts_HoldoutBeatsStickyUserProfileVariation() + { + InitializeLocalHoldoutsConfig(); + LocalHoldoutsConfig.GetGlobalHoldouts()[0].TrafficAllocation = new TrafficAllocation[0]; + + var userProfile = new UserProfile(TestUserId, new Dictionary + { + { "exp_rule_id_1", new Bucketing.Decision("exp1_var_1") }, + }); + var userProfileServiceMock = new Mock(); + userProfileServiceMock.Setup(ups => ups.Lookup(TestUserId)).Returns(userProfile.ToMap()); + + var decisionService = new DecisionService(new Bucketer(LoggerMock.Object), + new NoOpErrorHandler(), userProfileServiceMock.Object, LoggerMock.Object, null); + var userContext = new OptimizelyUserContext(LocalHoldoutsOptimizely, TestUserId, null, + new NoOpErrorHandler(), LoggerMock.Object); + + var result = decisionService.GetVariationsForFeatureList( + new List { LocalHoldoutsConfig.FeatureKeyMap["test_flag_2"] }, + userContext, LocalHoldoutsConfig, new UserAttributes(), + new OptimizelyDecideOption[0]); + + var decision = result[0].ResultObject; + Assert.AreEqual("experiment_rule_2", decision.Experiment?.Key); + Assert.AreEqual("variation_b", decision.Variation?.Key); + Assert.AreEqual(1, decision.HoldoutDecisions.Count); + Assert.AreEqual("local_holdout_exp_rule1", decision.HoldoutDecisions[0].Experiment.Key); + } + + [Test] + public void TestLocalHoldouts_GlobalExcludeTDHitThenLocalHitOrderedGlobalFirst() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(df => + { + var globalHoldout = df["holdouts"][0]; + globalHoldout["trafficAllocation"] = new JArray(new JObject + { + { "entityId", "gvar_1" }, + { "endOfRange", 10000 }, + }); + globalHoldout["excludeTargetedDeliveries"] = true; + ZeroTraffic(df, "localHoldouts", "holdout_local_rule2"); + }); + var config = optimizely.ProjectConfigManager.GetConfig() as DatafileProjectConfig; + var decisionService = new DecisionService(new Bucketer(LoggerMock.Object), + new NoOpErrorHandler(), null, LoggerMock.Object, null); + var userContext = optimizely.CreateUserContext(TestUserId); + + var result = decisionService.GetVariationsForFeatureList( + new List { config.FeatureKeyMap["test_flag_1"] }, userContext, config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + var featureDecision = result[0].ResultObject; + Assert.AreEqual("delivery_rule_2", featureDecision.Experiment?.Key); + Assert.AreEqual(2, featureDecision.HoldoutDecisions.Count); + Assert.AreEqual("global_holdout_2", featureDecision.HoldoutDecisions[0].Experiment.Key); + Assert.AreEqual("local_holdout_rule1", featureDecision.HoldoutDecisions[1].Experiment.Key); + + var decision = userContext.Decide("test_flag_1"); + + Assert.AreEqual("delivery_rule_2", decision.RuleKey); + VerifyHoldoutImpression("global_holdout_2", Times.Once()); + VerifyHoldoutImpression("local_holdout_rule1", Times.Once()); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestLocalHoldouts_DecisionErrorSuppressesHoldoutEvents() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(); + var config = optimizely.ProjectConfigManager.GetConfig() as DatafileProjectConfig; + var holdout = config.LocalHoldouts.First(h => h.Key == "local_holdout_exp_rule1"); + var errorDecision = new FeatureDecision(config.GetExperimentFromKey("experiment_rule_2"), + null, FeatureDecision.DECISION_SOURCE_FEATURE_TEST, null, error: true); + errorDecision.HoldoutDecisions.Add(new FeatureDecision(holdout, holdout.Variations[0], + FeatureDecision.DECISION_SOURCE_HOLDOUT)); + + var decisionServiceMock = new Mock(new Bucketer(LoggerMock.Object), + new NoOpErrorHandler(), null, LoggerMock.Object, null); + decisionServiceMock.Setup(ds => ds.GetVariationsForFeatureList( + It.IsAny>(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny())). + Returns(new List> + { + Result.NewResult(errorDecision, new DecisionReasons()), + }); + typeof(Optimizely).GetField("DecisionService", + System.Reflection.BindingFlags.NonPublic | + System.Reflection.BindingFlags.Instance). + SetValue(optimizely, decisionServiceMock.Object); + + var decision = optimizely.CreateUserContext(TestUserId).Decide("test_flag_2"); + + Assert.IsNull(decision.VariationKey); + Assert.IsFalse(decision.Enabled); + VerifyHoldoutImpression("local_holdout_exp_rule1", Times.Never()); + EventProcessorMock.Verify(ep => ep.Process(It.IsAny()), Times.Never()); + } + + [Test] + public void TestLocalHoldouts_LegacyApiSkipsHeldOutRuleWithoutHoldoutEvent() + { + var optimizely = CreateLocalHoldoutsOptimizelyWithMockedEvents(df => + { + df["sendFlagDecisions"] = true; + }); + + Assert.IsTrue(optimizely.IsFeatureEnabled("test_flag_1", TestUserId)); + Assert.IsTrue(optimizely.IsFeatureEnabled("test_flag_2", TestUserId)); + + EventProcessorMock.Verify(ep => ep.Process(It.Is(ie => + ie.Metadata.RuleType == FeatureDecision.DECISION_SOURCE_HOLDOUT)), Times.Never()); + } } } - diff --git a/OptimizelySDK/Bucketing/DecisionService.cs b/OptimizelySDK/Bucketing/DecisionService.cs index 9302793f..b287e0b0 100644 --- a/OptimizelySDK/Bucketing/DecisionService.cs +++ b/OptimizelySDK/Bucketing/DecisionService.cs @@ -623,7 +623,7 @@ UserProfile userProfile /// The feature flag the user wants to access. /// The user context. /// null if the user is not bucketed into the rollout or if the feature flag was not attached to a rollout. - /// otherwise the FeatureDecision entity + /// otherwise the FeatureDecision entity. If only local holdouts were hit, a decision with null Experiment/Variation carrying HoldoutDecisions. public virtual Result GetVariationForFeatureRollout( FeatureFlag featureFlag, OptimizelyUserContext user, @@ -666,6 +666,7 @@ ProjectConfig config var userId = user.GetUserId(); var attributes = user.GetAttributes(); + var holdoutHits = new List(); var index = 0; while (index < rolloutRulesLength) @@ -682,8 +683,8 @@ ProjectConfig config if (forcedDecisionResponse.ResultObject != null) { return Result.NewResult( - new FeatureDecision(rule, forcedDecisionResponse.ResultObject, null), - reasons); + WithHoldoutHits(new FeatureDecision(rule, forcedDecisionResponse.ResultObject, null), + holdoutHits), reasons); } var localHoldoutResult = EvaluateLocalHoldouts(rule.Id, user, config); @@ -692,8 +693,10 @@ ProjectConfig config { Logger.Log(LogLevel.INFO, reasons.AddInfo( - $"The user \"{userId}\" is bucketed into local holdout for delivery rule \"{rule.Key}\".")); - return Result.NewResult(localHoldoutResult.ResultObject, reasons); + $"The user \"{userId}\" is bucketed into local holdout for delivery rule \"{rule.Key}\". Skipping this rule.")); + holdoutHits.Add(localHoldoutResult.ResultObject); + index++; + continue; } // Regular decision @@ -728,8 +731,8 @@ ProjectConfig config $"User \"{userId}\" is in the traffic group of targeting rule \"{loggingKey}\".")); return Result.NewResult( - new FeatureDecision(rule, bucketedVariation.ResultObject, - FeatureDecision.DECISION_SOURCE_ROLLOUT), reasons); + WithHoldoutHits(new FeatureDecision(rule, bucketedVariation.ResultObject, + FeatureDecision.DECISION_SOURCE_ROLLOUT), holdoutHits), reasons); } else if (!everyoneElse) { @@ -751,7 +754,7 @@ ProjectConfig config index = skipToEveryoneElse ? rolloutRulesLength - 1 : index + 1; } - return Result.NullResult(reasons); + return HoldoutCarrierOrNull(holdoutHits, reasons); } @@ -762,7 +765,7 @@ ProjectConfig config /// User Identifier /// The user's attributes. This should be filtered to just attributes in the Datafile. /// null if the user is not bucketed into the rollout or if the feature flag was not attached to a rollout. - /// Otherwise the FeatureDecision entity + /// Otherwise the FeatureDecision entity. If only local holdouts were hit, a decision with null Experiment/Variation carrying HoldoutDecisions. public virtual Result GetVariationForFeatureExperiment( FeatureFlag featureFlag, OptimizelyUserContext user, @@ -788,6 +791,7 @@ public virtual Result GetVariationForFeatureExperiment( return Result.NullResult(reasons); } + var holdoutHits = new List(); foreach (var experimentId in featureFlag.ExperimentIds) { var experiment = config.GetExperimentFromId(experimentId); @@ -819,8 +823,9 @@ public virtual Result GetVariationForFeatureExperiment( { Logger.Log(LogLevel.INFO, reasons.AddInfo( - $"The user \"{userId}\" is bucketed into local holdout for experiment rule \"{experiment.Key}\".")); - return Result.NewResult(localHoldoutResult.ResultObject, reasons); + $"The user \"{userId}\" is bucketed into local holdout for experiment rule \"{experiment.Key}\". Skipping this rule.")); + holdoutHits.Add(localHoldoutResult.ResultObject); + continue; } var decisionResponse = GetVariation(experiment, user, config, options, @@ -858,14 +863,15 @@ public virtual Result GetVariationForFeatureExperiment( var featureDecision = new FeatureDecision(experiment, decisionVariation, FeatureDecision.DECISION_SOURCE_FEATURE_TEST); #endif - return Result.NewResult(featureDecision, reasons); + return Result.NewResult( + WithHoldoutHits(featureDecision, holdoutHits), reasons); } } Logger.Log(LogLevel.INFO, reasons.AddInfo( $"The user \"{userId}\" is not bucketed into any of the experiments on the feature \"{featureFlag.Key}\".")); - return Result.NullResult(reasons); + return HoldoutCarrierOrNull(holdoutHits, reasons); } /// @@ -916,6 +922,7 @@ public virtual Result GetDecisionForFlag( FeatureDecision globalHoldoutDecision = null; Holdout matchedHoldout = null; + var holdoutHits = new List(); var globalHoldouts = projectConfig.GetGlobalHoldouts(); foreach (var holdout in globalHoldouts) { @@ -931,6 +938,7 @@ public virtual Result GetDecisionForFlag( $"User \"{userId}\" is in holdout \"{holdout.Key}\" which excludes targeted deliveries. Targeted delivery rules will be evaluated normally.")); globalHoldoutDecision = holdoutDecision.ResultObject; matchedHoldout = holdout; + holdoutHits.Add(globalHoldoutDecision); } else { @@ -949,7 +957,11 @@ public virtual Result GetDecisionForFlag( filteredAttributes, projectConfig, options, userProfileTracker); reasons += experimentDecision.DecisionReasons; - if (experimentDecision.ResultObject != null) + if (IsHoldoutCarrier(experimentDecision.ResultObject)) + { + holdoutHits.AddRange(experimentDecision.ResultObject.HoldoutDecisions); + } + else if (experimentDecision.ResultObject != null) { return Result.NewResult(experimentDecision.ResultObject, reasons); } @@ -971,34 +983,35 @@ public virtual Result GetDecisionForFlag( var rolloutDecision = GetVariationForFeatureRollout(featureFlag, user, projectConfig); reasons += rolloutDecision.DecisionReasons; - if (rolloutDecision.ResultObject != null) + if (IsHoldoutCarrier(rolloutDecision.ResultObject)) + { + holdoutHits.AddRange(rolloutDecision.ResultObject.HoldoutDecisions); + } + else if (rolloutDecision.ResultObject != null) { Logger.Log(LogLevel.INFO, reasons.AddInfo( $"The user \"{userId}\" is bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); - if (globalHoldoutDecision != null) - { - rolloutDecision.ResultObject.HoldoutDecision = globalHoldoutDecision; - } + rolloutDecision.ResultObject.HoldoutDecisions.InsertRange(0, holdoutHits); return Result.NewResult(rolloutDecision.ResultObject, reasons); } + var nullDecision = WithHoldoutHits( + new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_ROLLOUT), holdoutHits); if (globalHoldoutDecision != null) { - var nullDecision = new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_ROLLOUT); - nullDecision.HoldoutDecision = globalHoldoutDecision; Logger.Log(LogLevel.INFO, reasons.AddInfo( $"The user \"{userId}\" is not bucketed into any targeted delivery for feature flag \"{featureFlag.Key}\". Holdout impression will be sent but holdout variation is not applied as fallback.")); - return Result.NewResult(nullDecision, reasons); + } + else + { + Logger.Log(LogLevel.INFO, + reasons.AddInfo( + $"The user \"{userId}\" is not bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); } - Logger.Log(LogLevel.INFO, - reasons.AddInfo( - $"The user \"{userId}\" is not bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); - return Result.NewResult( - new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_ROLLOUT), - reasons); + return Result.NewResult(nullDecision, reasons); } public virtual List> GetVariationsForFeatureList( @@ -1096,6 +1109,34 @@ private Result GetBucketingId(string userId, UserAttributes filteredAttr return Result.NewResult(bucketingId, reasons); } + private static FeatureDecision WithHoldoutHits(FeatureDecision decision, + List holdoutHits + ) + { + decision.HoldoutDecisions.AddRange(holdoutHits); + return decision; + } + + private static Result HoldoutCarrierOrNull( + List holdoutHits, + DecisionReasons reasons + ) + { + if (holdoutHits.Count == 0) + { + return Result.NullResult(reasons); + } + + return Result.NewResult( + WithHoldoutHits(new FeatureDecision(null, null, null), holdoutHits), reasons); + } + + private static bool IsHoldoutCarrier(FeatureDecision decision) + { + return decision != null && decision.Experiment == null && decision.Variation == null && + decision.Source == null && decision.HoldoutDecisions.Count > 0; + } + private Result EvaluateLocalHoldouts( string ruleId, OptimizelyUserContext user, diff --git a/OptimizelySDK/Entity/FeatureDecision.cs b/OptimizelySDK/Entity/FeatureDecision.cs index b340bb5a..330b041f 100644 --- a/OptimizelySDK/Entity/FeatureDecision.cs +++ b/OptimizelySDK/Entity/FeatureDecision.cs @@ -1,5 +1,5 @@ /* - * Copyright 2017, 2019, Optimizely + * Copyright 2017, 2019, 2026, Optimizely * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,9 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; + namespace OptimizelySDK.Entity { public class FeatureDecision @@ -26,7 +29,24 @@ public class FeatureDecision public string Source { get; } public string CmabUuid { get; } public bool Error { get; } - public FeatureDecision HoldoutDecision { get; set; } + public List HoldoutDecisions { get; } = new List(); + + [Obsolete("Use HoldoutDecisions instead.")] + public FeatureDecision HoldoutDecision + { + get + { + return HoldoutDecisions.Count > 0 ? HoldoutDecisions[0] : null; + } + set + { + HoldoutDecisions.Clear(); + if (value != null) + { + HoldoutDecisions.Add(value); + } + } + } public FeatureDecision(ExperimentCore experiment, Variation variation, string source, string cmabUuid = null, bool error = false) { diff --git a/OptimizelySDK/Optimizely.cs b/OptimizelySDK/Optimizely.cs index e9040710..098bdc5f 100644 --- a/OptimizelySDK/Optimizely.cs +++ b/OptimizelySDK/Optimizely.cs @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024, Optimizely + * Copyright 2017-2024, 2026, Optimizely * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use file except in compliance with the License. @@ -1126,21 +1126,10 @@ ProjectConfig projectConfig var ruleKey = flagDecision.Experiment?.Key; var decisionEventDispatched = false; - if (flagDecision?.HoldoutDecision != null && !allOptions.Contains(OptimizelyDecideOption.DISABLE_DECISION_EVENT)) - { - decisionEventDispatched = SendImpressionEvent( - flagDecision.HoldoutDecision.Experiment, - flagDecision.HoldoutDecision.Variation, - userId, user.GetAttributes(), projectConfig, - flagKey, FeatureDecision.DECISION_SOURCE_HOLDOUT, - flagDecision.HoldoutDecision.Variation?.FeatureEnabled ?? false -#if USE_CMAB - , null -#endif - ) || decisionEventDispatched; - } if (!allOptions.Contains(OptimizelyDecideOption.DISABLE_DECISION_EVENT)) { + decisionEventDispatched = SendHoldoutImpressions(user, flagKey, flagDecision, + projectConfig); decisionEventDispatched = SendImpressionEvent( flagDecision.Experiment, flagDecision.Variation, @@ -1227,6 +1216,28 @@ private OptimizelyDecideOption[] GetAllOptions(OptimizelyDecideOption[] options) return copiedOptions; } + private bool SendHoldoutImpressions(OptimizelyUserContext user, string flagKey, + FeatureDecision flagDecision, ProjectConfig projectConfig + ) + { + var dispatched = false; + foreach (var holdoutDecision in flagDecision.HoldoutDecisions) + { + dispatched = SendImpressionEvent( + holdoutDecision.Experiment, + holdoutDecision.Variation, + user.GetUserId(), user.GetAttributes(), projectConfig, + flagKey, FeatureDecision.DECISION_SOURCE_HOLDOUT, + holdoutDecision.Variation?.FeatureEnabled ?? false +#if USE_CMAB + , null +#endif + ) || dispatched; + } + + return dispatched; + } + /// /// Sends impression event. ///