Mobile A/B Test Calculator: Sample Size & Statistical Significance

Plan mobile A/B tests correctly: calculate required sample sizes, understand statistical significance at 95% confidence, set realistic MDE targets, and integrate Firebase A/B Testing or Optimizely with your React Native app.

Sample Size Reference Table

Required sample size per variant (95% confidence, 80% power) at different baseline conversion rates and MDE values:

Firebase A/B Testing Setup for React Native

npm install @react-native-firebase/app @react-native-firebase/remote-config @react-native-firebase/analytics

// hooks/useExperiment.ts
import remoteConfig from '@react-native-firebase/remote-config';
import analytics from '@react-native-firebase/analytics';

export function useExperiment(experimentKey: string, defaultValue: string) {
  const [variant, setVariant] = useState(defaultValue);

  useEffect(() => {
    // Fetch remote config values (Firebase assigns user to variant)
    remoteConfig()
      .setConfigSettings({ minimumFetchIntervalMillis: 300000 })
      .then(() => remoteConfig().fetchAndActivate())
      .then(() => {
        const value = remoteConfig().getString(experimentKey);
        setVariant(value || defaultValue);

        // Log exposure event so Firebase knows this user saw the experiment
        analytics().logEvent('experiment_exposure', {
          experiment_key: experimentKey,
          variant: value,
        });
      });
  }, [experimentKey]);

  return variant;
}

// Usage in a component
const CheckoutScreen = () => {
  const ctaVariant = useExperiment('checkout_cta_text', 'Buy Now');

  const handlePurchase = async () => {
    // Log the goal event — Firebase A/B Testing tracks this
    await analytics().logEvent('purchase', { value: 29.99, currency: 'USD' });
  };

  return (
    <Button title={ctaVariant} onPress={handlePurchase} />
  );
};

Lift Calculation Formula

Calculate relative lift and whether your result is practically significant:

// Calculate statistical significance manually
function calculateSignificance(
  controlUsers: number,
  controlConversions: number,
  variantUsers: number,
  variantConversions: number
) {
  const controlRate = controlConversions / controlUsers;
  const variantRate = variantConversions / variantUsers;

  const relativeLift = ((variantRate - controlRate) / controlRate) * 100;

  const pooledRate = (controlConversions + variantConversions) / (controlUsers + variantUsers);
  const standardError = Math.sqrt(
    pooledRate * (1 - pooledRate) * (1 / controlUsers + 1 / variantUsers)
  );

  const zScore = (variantRate - controlRate) / standardError;
  const pValue = 2 * (1 - normalCDF(Math.abs(zScore)));

  return {
    controlRate: (controlRate * 100).toFixed(2) + '%',
    variantRate: (variantRate * 100).toFixed(2) + '%',
    relativeLift: relativeLift.toFixed(1) + '%',
    zScore: zScore.toFixed(2),
    pValue: pValue.toFixed(4),
    significant: pValue < 0.05,
  };
}