Statistical testing

Binder

Summary: this tutorial explains how to perform various statistical tests with FSharp.Stats.

Table of contents

FSharp.Stats provides hypothesis tests for different applications. A hypothesis test is a statistical test that is used to determine whether there is enough evidence in a sample of data to infer that a certain condition is true for the entire population. A hypothesis test examines two opposing hypotheses about a population: the null hypothesis and the alternative hypothesis.

Test Statistics

T-Test

By using a t test a difference of means can be evaluated. There are different kinds of t test designs implemented in FSharp.Stats.

  1. One sample t test:

    • Is the population mean equal to the value of H0?
    • e.g. “Is my grade different from the distribution mean?”
  2. Two sample t test with equal variances:

    • Prerequisite: equal variances
    • Is the mean of Population A equal to the mean of Population B?
    • 2.1 unpaired t test:

      • e.g.: Does the cell size differ between wildtype and mutant?
    • 2.2 paired t test:

      • e.g.: Does the medication influence the blood pressure? Measurement of the same persons before and after medication.
  3. Two sample t test with unequal variances

    • Welch test (unpaired)

Case 1: One sample t test

open FSharp.Stats
open FSharp.Stats.Testing

let sampleA = vector [|4.5; 5.1; 4.8; 4.4; 5.0|]

// calculates a one sample t test with a given sample and the fixed value the sample should be compared with
let oneSampleTTest = TTest.oneSample sampleA 5.

(*
    The test returns no significant p value:
    oneSampleTTest.PValue = 0.1533
*)

Case 2: Two sample t test with equal variances (unpaired)

A standard two sample t test expects the samples to be taken from populations with equal standard deviations. Violations of this requirement result in an inflated false positive rate.

let sample1 = vector [|4.9;5.0;6.7;4.8;5.2|]
let sample2 = vector [|3.9;4.9;3.8;4.5;4.5|]

let twoSampleTTest = TTest.twoSample true sample1 sample2

(*
    The test returns a significant p value:
    twoSampleTTest.PValue = 0.0396
*)

Case 3: Two sample t test with equal variances (paired)

A paired t-test is used to compare two population means where you have two samples in which observations in one sample can be paired with observations in the other sample. Examples of where this might occur are:

  • Before-and-after observations on the same subjects (e.g. students’ diagnostic test results before and after a particular module or course).
  • A comparison of two different methods of measurement or two different treatments where the measurements/treatments are applied to the same subjects (e.g. blood pressure measurements using a stethoscope and a dynamap).
let sampleP1 = vector [18.;21.;16.;22.;19.;24.;17.;21.;23.;18.;14.;16.;16.;19.;18.;20.;12.;22.;15.;17.;]
let sampleP2 = vector [22.;25.;17.;24.;16.;29.;20.;23.;19.;20.;15.;15.;18.;26.;18.;24.;18.;25.;19.;16.;]

let paired = TTest.twoSamplePaired sampleP1 sampleP2

(*
    The test returns a significant p value:
    paired.PValue = 0.00439
*)

Case 4: Two sample t test with unequal variances (Welch test)

If you are unsure about the nature of the underlying population, you may ask if the theoretical population distributions you want to compare do have the same standard deviations.

If not the welch test can serve as an appropriate hypothesis test for mean differences.

let sampleW1 = vector [0.8;0.9;1.0;1.1;1.2]
let sampleW2 = vector [0.8;1.1;1.3;1.5;2.0]

let welch = TTest.twoSample false sampleW1 sampleW2

(*
    The test returns a not significant p value:
    welch.PValue = 0.1725626595
*)

Anova

let dataOneWay =
    [|
    [|0.28551035; 0.338524035; 0.088313218; 0.205930807; 0.363240102;|];
    [|0.52173913; 0.763358779; 0.32546786; 0.425305688; 0.378071834; |];
    [|0.989119683; 1.192718142; 0.788288288; 0.549176236; 0.544588155;|];
    [|1.26705653; 1.625320787; 1.266108976; 1.154187629; 1.268498943; 1.069518717;|];
    |]

let contrastMatrix = 
    [| 
    [|1.0;-1.0;0.0;0.0;|]
    [|1.0;0.0;-1.0;0.0;|]
    [|1.0;0.0;0.0;-1.0;|]
    [|0.0;1.0;-1.0;0.0;|]
    [|0.0;1.0;0.0;-1.0;|]
    [|0.0;0.0;1.0;-1.0;|]
    |]

let oneWayResult = Anova.oneWayAnova dataOneWay
{ Factor = { DegreesOfFreedom = 3.0
             MeanSquares = 1.081456009
             Significance = 9.257958872e-07
             Source = BetweenGroups
             Statistic = 27.59425833
             SumOfSquares = 3.244368027 }
  Error = { DegreesOfFreedom = 17.0
            MeanSquares = 0.03919134177
            Significance = nan
            Source = WithinGroups
            Statistic = nan
            SumOfSquares = 0.6662528101 }
  Total = { DegreesOfFreedom = 20.0
            MeanSquares = 0.1955310419
            Significance = nan
            Source = Total
            Statistic = nan
            SumOfSquares = 3.910620837 } }
(*
anovaResult.Factor.Statistic = 27.594
The factor statistic indicates how much more variability there is between the the samples 
than within the samples.
anovaResult.Factor.PValue = 9.257958872e-07
A strong significant p value in the factor field indicates that one or more means differ from each other
*)
// http://astatsa.com/OneWay_Anova_with_TukeyHSD/
// https://www.wessa.net/rwasp_Two%20Factor%20ANOVA.wasp

let data =
    [
        (0.28, 'A', 'M');
        (0.95, 'A', 'M');
        (0.96, 'A', 'M');
        (0.97, 'A', 'M');
        (0.40, 'A', 'M');
        (0.18, 'A', 'M');
        (0.12, 'A', 'M');
        (0.62, 'A', 'M');
        (1.81, 'A', 'F');
        (1.51, 'A', 'F');
        (1.41, 'A', 'F');
        (1.39, 'A', 'F');
        (1.20, 'A', 'F');
        (1.55, 'A', 'F');
        (1.48, 'A', 'F');
        (1.25, 'A', 'F');
        (0.95, 'B', 'M');
        (1.33, 'B', 'M');
        (0.92, 'B', 'M');
        (0.85, 'B', 'M');
        (1.06, 'B', 'M');
        (0.69, 'B', 'M');
        (0.70, 'B', 'M');
        (0.79, 'B', 'M');
        (2.93, 'B', 'F');
        (3.24, 'B', 'F');
        (3.42, 'B', 'F');
        (2.79, 'B', 'F');
        (2.54, 'B', 'F');
        (3.28, 'B', 'F');
        (2.80, 'B', 'F');
        (3.40, 'B', 'F');
    ]
    //f1
    |> Seq.groupBy (fun (v,f1,f2) -> f1)
    //f2
    |> Seq.map (fun (k,vls) -> 
        vls 
        |> Seq.groupBy (fun (v,f1,f2) -> f2)
        |> Seq.map (fun (k,vls') -> vls' |> Seq.map (fun (v,f1,f2) -> v) |> Seq.toArray)
        |> Seq.toArray
        ) 
    |> Seq.toArray
    

Anova.twoWayANOVA Anova.TwoWayAnovaModel.Mixed data

// http://statweb.stanford.edu/~susan/courses/s141/exanova.pdf
// http://scistatcalc.blogspot.de/2013/11/two-factor-anova-test-calculator.html#

let data' =
    [|
      // super
        // cold super
      [|[|4.;5.;6.;5.;|];
        // warm super
        [|7.;9.;8.;12.;|];
        // hot super
        [|10.;12.;11.;9.; |]|];
      // best 
        // cold best
      [|[|6.;6.;4.;4.;|];
        // warm best
        [|13.;15.;12.;12.;|];
        // hot best
        [|12.;13.;10.;13.;|]|]
    |]

Anova.twoWayANOVA Anova.TwoWayAnovaModel.Mixed data'

F-Test

The F-test is a method to determine if the variances of two samples are homogeneous. Also, ANOVA (analysis of variance) is based on the F-test and is used for the comparison of more than two samples. Knowing if your variances are equal (H0 is true) helps to decide which test should be performed next. To determine if your variances are in fact equal you can perform a F-test.

Prerequisites:

  • population distributions are normally distributed
  • samples are independent

Possible further tests:

  • two sample t-test with equal variances
  • two sample t-test with unequal variances (Welch-test)
  • ...

Note that there is no information about the direction of variance difference (e.g. Zimmermann 2004). In this implemented test the larger variance is always the numerator, therefore the comparison to Fdf1,df2,1-(alpha/2) is used for a two sided test.

Important note: The effectiveness of a preliminary test of variances is discussed. The F-test is extremely sensitive to normality-violations, and even if the samples follow a normal distribution, it often does not detect situations where a t-test should be avoided.

References:

  • Jürgen Bortz & Christof Schuster, Statistik für Human- und Sozialwissenschaftler (2010) Chapter 8.6
  • Markowski, Conditions for the Effectiveness of a Preliminary Test of Variance (1990)
  • Shlomo S. Sawilowsky, The Probable Difference Between Two Means When σ12≠σ22 (2002)
  • Ronald Ley, F curves have two tails but the F test is a one-tailed two-tailed test (1979) + Reviewer comments
  • Donald W. Zimmermann, A note on preliminary tests of equality ofvariances (2004)

F-Test from data:

let sampleFA = vector [|5.0; 6.0; 5.8; 5.7|] 
let sampleFB = vector [|3.5; 3.7; 4.0; 3.3; 3.6|]

// comparison of sample variances 
let fTestFromData = FTest.testVariances sampleFA sampleFB
(* 
    { Statistic = 2.823383085
    DegreesOfFreedom1 = 3.0
    DegreesOfFreedom2 = 4.0
    PValue = 0.1708599931 }
    Using a significance level of 0.05 the sample variances do differ significantly.

*F-Test from given parameters:*
*)

// sample properties are given as (variance,degree of freedom) 
let sampleF1 = (0.1, 15.)
let sampleF2 = (0.05, 7.)

// comparison of sample variances 
let fTestFromParameters = FTest.testVariancesFromVarAndDof sampleF1 sampleF2
{ Statistic = 2.0
  DegreesOfFreedom1 = 15.0
  DegreesOfFreedom2 = 7.0
  PValue = 0.17963663
  PValueTwoTailed = 0.35927326 }

Using a significance level of 0.05 the sample variances do differ significantly.

H Test

The H test is also known as Kruskal-Wallis one-way analysis-of-variance-by-ranks and is the non-parametric equivalent of one-way ANOVA. It is a non-parametric test for comparing the means of more than two independent samples (equal or different sample size), and therefore is an extension of Wilcoxon-Mann-Whitney two sample test. Testing with H test gives information whether the samples are from the same distribution.

A benefit of the H test is that it does not require normal distribution of the samples. The downside is that there is no information which samples are different from each other, or how many differences occur. For further investigation a post hoc test is recommended.

The distribution of the H test statistic is approximated by chi-square distribution with degrees of freedom = sample count - 1.

The implemented H-test is testing for duplicate values in the data. Duplicates lead to ties in the ranking, and are corrected by using a correction factor.

Prerequisites:

  • random and independent samples
  • observations are from populations with same shape of distribution
  • nominal scale, ordinal scale, ratio scale or interval scale data

References:

  • E. Ostertagová, Methodology and Application of the Kruskal-Wallis Test (2014)
  • Y. Chan, RP Walmsley, Learning and understanding the Kruskal-Wallis one-way analysis-of-variance-by-ranks test for differences among three or more independent groups (1997)
let groupA = [44.;44.;54.;32.;21.;28.]
let groupB = [70.;77.;48.;64.;71.;75.]
let groupC = [80.;76.;34.;80.;73.;80.] 
let samples = [groupA;groupB;groupC]

// calculation of the H test 
let hResult = HTest.createHTest samples 
{ Statistic = 9.781466113
  DegreesOfFreedom = 2.0
  PValueLeft = 0.9924840891
  PValueRight = 0.007515910866
  PValue = 0.01503182173 }

PValueRight is significant at a alpha level of 0.05

A suitable post hoc test for H tests is Dunn's test.

Chi-Squared Test

WIP

Bartlett

WIP

PostHoc

ANOVA provides the user with a global statement if samples differ from each other. It does not provide detailed information regarding differences of the single samples.

If the H0 hypothesis is neglected (so significant differences are assumed), a post hoc test (multiple comparison test) allows the pairwise comparison of the individual groups.

Reference: What is the proper way to apply the multiple comparison test?, Sangseok Lee and Dong Kyu Lee, 2018

Fisher's LSD

The simplest method is Fisher's least significant difference (Fisher's LSD). It calculates Student's t tests for all pairwise comparisons. But instead of estimating the variance for each sample separately it takes all groups into account. Violations of the homogeneity of variances reduce the test power. Since no correction for multiple comparisons is performed, the resulting p values must be corrected (for example with Benjamini-Hochberg method).

Important: Fishers LSD is dependent to a significant ANOVA (ANOVA-protected post-hoc test).

open PostHoc

let lsd = Testing.PostHoc.fishersLSD contrastMatrix dataOneWay

// For multi comparison correction, the p values are adjusted by the Benjamini-Hochberg approach
let (index,pValue,pValueAdj) = 
    lsd
    |> Testing.MultipleTesting.benjaminiHochbergFDRBy (fun x -> x,x.Significance)
    |> List.sortBy (fun (x,_) -> x.Index)
    |> List.map (fun (x,pValAdj) -> x.Index, x.Significance, pValAdj)
    |> List.unzip3

open Plotly.NET

//some axis styling
let myAxis title = Axis.LinearAxis.init(Title=title,Mirror=StyleParam.Mirror.All,Ticks=StyleParam.TickOptions.Inside,Showgrid=false,Showline=true,Zeroline=true)
let myAxisRange name range = Axis.LinearAxis.init(Title=name,Range=StyleParam.Range.MinMax(range), Mirror=StyleParam.Mirror.All,Ticks=StyleParam.TickOptions.Inside,Showgrid=false,Showline=true)
let styleChart x y chart = chart |> Chart.withX_Axis (myAxis x) |> Chart.withY_Axis (myAxis y)
let styleChartRange x y rx ry chart = chart |> Chart.withX_Axis (myAxisRange x rx) |> Chart.withY_Axis (myAxisRange y ry)

let lsdCorrected =
    let header = ["<b>Contrast index</b>";"<b>p Value</b>";"<b>p Value adj</b>"]
    let rows = 
        [
            [sprintf "%i" index.[0];sprintf "%.6f" pValue.[0];sprintf "%.6f" pValueAdj.[0];]    
            [sprintf "%i" index.[1];sprintf "%.6f" pValue.[1];sprintf "%.6f" pValueAdj.[1];]    
            [sprintf "%i" index.[2];sprintf "%.6f" pValue.[2];sprintf "%.6f" pValueAdj.[2];]       
            [sprintf "%i" index.[3];sprintf "%.6f" pValue.[3];sprintf "%.6f" pValueAdj.[3];]       
            [sprintf "%i" index.[4];sprintf "%.6f" pValue.[4];sprintf "%.6f" pValueAdj.[4];]    
            [sprintf "%i" index.[5];sprintf "%.6f" pValue.[5];sprintf "%.6f" pValueAdj.[5];]
        ]
    
    Chart.Table(
        header, 
        rows,
        ColorHeader = "#45546a",
        ColorCells = ["#deebf7";"lightgrey"],
        FontHeader = Font.init(Color="white")
        )

Hays

Testing.PostHoc.hays contrastMatrix dataOneWay 


//let m1 = Seq.mean dataOneWay.[0]
//let m2 = Seq.mean dataOneWay.[1]

//let s1 = Seq.sum dataOneWay.[0]
//let s2 = Seq.sum dataOneWay.[1]
//let s3 = Seq.sum dataOneWay.[2]


//let d = (m1-m2)**2.0
//d / (3.926003843 * (1./5. + 1./5.))

// http://www.statisticslectures.com/topics/posthoconewayanova/
let dmg = 
    [|
     [|9.;8.;7.;8.;8.;9.;8.;|];
     [|7.;6.;6.;7.;8.;7.;6.;|];
     [|4.;3.;2.;3.;4.;3.;2.;|] ; 
    |]

let contrastMatrixDmg = 
    [| 
     [|1.0;-1.0;0.0;|] ; 
     [|1.0;0.0;-1.0;|] ; 
     [|0.0;1.0;-1.0;|]   
    |]


Testing.PostHoc.hays contrastMatrixDmg dmg 

Anova.oneWayAnova dmg

Tukey HSD

Tukeys honestly significant difference (HSD) can be used to inspect a significant ANOVA result for underlying causes.

Important note: Various discussions question the application of Tukeys HSD only to significant ANOVA results (Anova-protected post-hoc test), since Tukeys HSD already controls for multiple testing. Inform yourself prior to using an post hoc test appropriate to your experimental design. Fishers LSD however is dependent to a significant ANOVA.

Using this post hoc test you can determine which of the means differ significantly from each other. In the classis Tukeys HSD approach, the population variances are pooled for a more robust estimation (especially with small sample sizes). If the population variances differ, Tukeys HSD is not appropriate.

The implemented Tukey-Kramer-Method can be applied on unequal sample sizes and estimates the variance based on the means to compare. The Tukey-Kramer method can be used as stand-alone method for comparing multiple means.

A comparison of ANOVA and Tukey-Kramer-HSD with simulations for the robustness of normality-violations of the data can be found in:

Robustness of the ANOVA and Tukey-Kramer Statistical Tests, Wade C. Driscoll, Computers ind. Engng Vol 31, No. 1/2, pp. 265 - 268, 1996

Multiple-to-one comparisons can alternatively performed with Dunnet's test, which was designed for performing k-1 tests, while Tukeys HSD performs k((k-1)/2) tests.

Task: It should be tested if one or more means of samples taken from different mutants differ from each other.

// Example values from: https://brownmath.com/stat/anova1.htm
let hsdExample = 
    [|
        [|64.; 72.; 68.; 77.; 56.; 95.;|] // sample of mutant/drug/factor 1 
        [|78.; 91.; 97.; 82.; 85.; 77.;|] // sample of mutant/drug/factor 2 
        [|75.; 93.; 78.; 71.; 63.; 76.;|] // sample of mutant/drug/factor 3 
        [|55.; 66.; 49.; 64.; 70.; 68.;|] // sample of mutant/drug/factor 4    
    |]

let anovaResult = Anova.oneWayAnova hsdExample

(*
    anovaResult.Factor.Statistic = 5.41
    The factor statistic indicates how much more variability there is between the the samples 
    than within the samples.
    anovaResult.Factor.PValue = 0.0069
    A significant p value in the factor field indicates that one or more means differ from each other
*)

For tukey HSD calculations you need a contrast matrix, that defines the groups you want to compare for more detailed information.

Every contrast has as many entries as there are groups (samples). The groups, that should be compared are labelled with -1 or 1 respectively.

// In this contrast matrix every possible scenario is covered.
let contrastMatrixHSD = 
    [| 
    [|1.;-1.; 0.; 0.;|] // sample 1 is compared to sample 2
    [|1.; 0.;-1.; 0.;|] // sample 1 is compared to sample 3
    [|1.; 0.; 0.;-1.;|] 
    [|0.; 1.;-1.; 0.;|]
    [|0.; 1.; 0.;-1.;|]
    [|0.; 0.; 1.;-1.;|] // sample 3 is compared to sample 4
    |]

let hsdResult = tukeyHSD contrastMatrixHSD hsdExample 
"index	meanDiff	qStatistic	pValue
    0	13.000000	3.170103	0.146193
    1	4.000000	0.975416	0.899806
    2	10.000000	2.438541	0.337815
    3	9.000000	2.194687	0.427072
    4	23.000000	5.608644	0.003906
    5	14.000000	3.413957	0.106557"
(*
    For every generated contrast an output p value is calculated.
    e.g.
    hsdResult.[0].Significance = 0.0364
    hsdResult.[1].Significance = 0.4983 
    hsdResult.[2].Significance = 0.1001
    hsdResult.[3].Significance = 0.1364
    hsdResult.[4].Significance = 0.0008
    hsdResult.[5].Significance = 0.0255
*)

//TTest.twoSample true (vector hsdExample.[0]) (vector hsdExample.[1])
//TTest.twoSample true (vector hsdExample.[0]) (vector hsdExample.[2])
//TTest.twoSample true (vector hsdExample.[0]) (vector hsdExample.[3])
//TTest.twoSample true (vector hsdExample.[1]) (vector hsdExample.[2])
//TTest.twoSample true (vector hsdExample.[1]) (vector hsdExample.[3])
//TTest.twoSample true (vector hsdExample.[2]) (vector hsdExample.[3])

Dunnett's (t) test

When there is one control group which should be compared with all treatment-groups, Tukeys HSD would lead to an explosion of comparisons if the number of conditions increases. If just the comparison of each treatment to an control is required you can use Dunnett's test. It is a multiple-to-one post hoc test for homoscedastic samples with equal variance that has a higher power than Tukey's HSD since fewer comparisons have to be performed, and therefore the Confidence limits are wider than necessary. "ANOVA is not a necessary part of the multiple comparisons procedure" (Dunnett, 1964).

Note: Dunnett's test is designed for equal group sizes and will only provide approximate values when group sizes differ (Dunnett 1955).

Reference:

  • A Multiple Comparison Procedure for Comparing Several Treatments with a Control; CW Dunnett; Journal of the American Statistical Association; Dec. 1955
  • New Tables for Multiple Comparisons with a Control; CW Dunnett; Biometrics; Sep. 1964
let dunnetExample = 
    [|
        [|1.84;2.49;1.50;2.42;|]
        [|2.43;1.85;2.42;2.73;|]
        [|3.95;3.67;3.23;2.31;|]
        [|3.21;3.20;2.32;3.30;|]
        //only gives aproximate results when group sizes are unequal
        [|3.21;3.13;2.32;3.30;3.20;2.42;|]
    |]

//first sample serves as control
let dunnetContrastMatrix = 
    [|                
        [|-1.;1.;0.;0.;0.|]
        [|-1.;0.;1.;0.;0.|]
        [|-1.;0.;0.;1.;0.|]
        [|-1.;0.;0.;0.;1.|]
    |]

let dunnettResult = 
    PostHoc.dunnetts dunnetContrastMatrix dunnetExample Tables.dunnetts095
No value returned by any evaluator

Fisher Hotelling

let d1 = [159.;179.;100.;45.;384.;230.;100.;320.;80.;220.;320.;210.;]
let d2 = [14.4;15.2;11.3;2.5;22.7;14.9;1.41;15.81;4.19;15.39;17.25;9.52; ]
    
Testing.FisherHotelling.test d1 d2

Multiple testing

When conducting multiple hypothesis test the α-error accumulates. This is because the p value just describes the probability for a false positive for one single test. If you perform 10 t-test at an α level of 0.05, the probability of getting a significant result by chance is 40.1% [ (1-(1-α)k ].

let aErrorAcc = 
    [1. .. 100.]
    |> List.map (fun x -> x,(1. - 0.95**x))
    |> Chart.Line
    |> styleChart "number of tests (k)" "probability of at least one false positive test"

To compensate this inflation, several multiple testing corrections were published. The most conservative method is the Bonferroni correction, where the used α level is divided by the number of performed tests.

A modern correction approach is the Benjamini-Hochberg method also known as FDR (false discovery rate).

Benjamini-Hochberg

let pValues =
    [|
        0.000002;0.000048;0.000096;0.000096;0.000351;0.000368;0.000368;0.000371;0.000383;0.000383;0.000705;0.000705;0.000705;0.000705;0.000739;0.00101;0.001234;0.001509;0.001509;0.001509;0.001509;0.001686;0.001686;0.001964;0.001964;0.001964;0.001964;0.001964;0.001964;0.001964;0.002057;0.002295;0.002662;0.002662;
        0.002662;0.002662;0.002662;0.002662;0.002662;0.002672;0.002714;0.002922;0.00348;0.004066;0.004176;0.004176;0.004562;0.004562;0.005848;0.005848;0.006277;0.007024;0.007614;0.007614;0.007614;0.007614;0.007614;0.00979;0.01049;0.01049;0.012498;0.012498;0.012498;0.017908;0.018822;0.019003;0.019003;0.019003;
        0.020234;0.02038;0.021317;0.023282;0.026069;0.026773;0.027255;0.027255;0.027255;0.027255;0.0274;0.030338;0.03128;0.034516;0.034516;0.037267;0.037267;0.040359;0.042706;0.043506;0.04513;0.04513;0.047135;0.049261;0.049261;0.049261;0.049261;0.049333;0.050457;0.052112;0.052476;0.060504;0.063031;0.063031;
        0.063031;0.063031;0.065316;0.065316;0.066751;0.067688;0.069676;0.073043;0.078139;0.078594;0.078594;0.095867;0.098913;0.102606;0.102606;0.102606;0.107444;0.116213;0.126098;0.135099;0.135099;0.159786;0.179654;0.199372;0.203542;0.203542;0.211249;0.211968;0.226611;0.228287;0.238719;0.247204;0.263942;
        0.263942;0.289175;0.306064;0.330191;0.330191;0.340904;0.343869;0.350009;0.355614;0.355614;0.359354;0.386018;0.386018;0.434486;0.438791;0.464694;0.471015;0.4715;0.479307;0.490157;0.505652;0.539465;0.539465;0.558338;0.558338;0.601991;0.61052;0.634365;0.637835;0.677506;0.678222;0.727881;0.748533;
        0.753718;0.758701;0.810979;0.838771;0.854833;0.872159;0.878727;0.890621;0.916361;0.954779;0.98181;0.985365;0.986261;0.98958;0.99861;0.99861;0.999602;0.999895
    |] |> Array.sort

let pValsAdj =
    MultipleTesting.benjaminiHochbergFDRBy (fun x -> x,x) pValues
    |> List.rev

let bhValues =
    [
        Chart.Line(pValues,pValues,Name="diagonal")
        Chart.Line(pValsAdj,Name="adj")
    ]
    |> Chart.Combine
    |> styleChartRange "pValue" "BH corrected pValue" (0.,1.) (0.,1.)

Q Value

let pi0 = 
    pValues
    |> MultipleTesting.Qvalues.pi0Bootstrap 

let qValues = 
    pValues
    |> MultipleTesting.Qvalues.ofPValues pi0

let qValuesRob =
    pValues
    |> MultipleTesting.Qvalues.ofPValuesRobust pi0 

let qChart =    
    [
        Chart.Line(pValues,qValues,Name="qValue")
        Chart.Line(pValues,qValuesRob,Name="qValueRobust")
    ]
    |> Chart.Combine
    |> styleChartRange "pValues" "qValues" (0.,1.) (0.,1.)

let qHisto =
    [
        Chart.Histogram(pValues,Xbins=Bins.init(0.,1.,0.05),Name="pValues",HistNorm=StyleParam.HistNorm.Density)
        Chart.Line([(0.,pi0);(1.,pi0)],Name="pi<sub>0</sub>",Dash=StyleParam.DrawingStyle.Dash)
    ]
    |> Chart.Combine
    |> styleChart "p value" "density"
Multiple items
namespace FSharp

--------------------
namespace Microsoft.FSharp
namespace FSharp.Stats
namespace FSharp.Stats.Testing
val sampleA : Vector<float>
Multiple items
val vector : l:seq<float> -> Vector<float>

--------------------
type vector = Vector<float>
val oneSampleTTest : TestStatistics.TTestStatistics
module TTest

from FSharp.Stats.Testing
val oneSample : sample1:Vector<float> -> mu:float -> TestStatistics.TTestStatistics
val sample1 : Vector<float>
val sample2 : Vector<float>
val twoSampleTTest : TestStatistics.TTestStatistics
val twoSample : assumeEqualVariances:bool -> sample1:Vector<float> -> sample2:Vector<float> -> TestStatistics.TTestStatistics
val sampleP1 : Vector<float>
val sampleP2 : Vector<float>
val paired : TestStatistics.TTestStatistics
val twoSamplePaired : sample1:Vector<float> -> sample2:Vector<float> -> TestStatistics.TTestStatistics
val sampleW1 : Vector<float>
val sampleW2 : Vector<float>
val welch : TestStatistics.TTestStatistics
val dataOneWay : float [] []
val contrastMatrix : float [] []
val oneWayResult : Anova.OneWayAnovaVariationSources
module Anova

from FSharp.Stats.Testing
val oneWayAnova : samples:seq<#seq<float>> -> Anova.OneWayAnovaVariationSources
val data : float [] [] []
Multiple items
module Seq

from FSharp.Stats

--------------------
module Seq

from Microsoft.FSharp.Collections
val groupBy : projection:('T -> 'Key) -> source:seq<'T> -> seq<'Key * seq<'T>> (requires equality)
val v : float
val f1 : char
val f2 : char
val map : mapping:('T -> 'U) -> source:seq<'T> -> seq<'U>
val k : char
val vls : seq<float * char * char>
val vls' : seq<float * char * char>
val toArray : source:seq<'T> -> 'T []
val twoWayANOVA : anovaType:Anova.TwoWayAnovaModel -> samples:float array array array -> Anova.TwoWayAnovaVariationSources
type TwoWayAnovaModel =
  | Fixed
  | Mixed
  | Random
union case Anova.TwoWayAnovaModel.Mixed: Anova.TwoWayAnovaModel
val data' : float [] [] []
val sampleFA : Vector<float>
val sampleFB : Vector<float>
val fTestFromData : TestStatistics.FTestStatistics
module FTest

from FSharp.Stats.Testing
val testVariances : data1:Vector<float> -> data2:Vector<float> -> TestStatistics.FTestStatistics
val sampleF1 : float * float
val sampleF2 : float * float
val fTestFromParameters : TestStatistics.FTestStatistics
val testVariancesFromVarAndDof : var1:float * df1:float -> var2:float * df2:float -> TestStatistics.FTestStatistics
val groupA : float list
val groupB : float list
val groupC : float list
val samples : float list list
val hResult : TestStatistics.ChiSquareStatistics
module HTest

from FSharp.Stats.Testing
val createHTest : samples:seq<#seq<float>> -> TestStatistics.ChiSquareStatistics
module PostHoc

from FSharp.Stats.Testing
val lsd : Contrast []
val fishersLSD : contrastMatrix:float [] [] -> data:float [] [] -> Contrast []
val index : int list
val pValue : float list
val pValueAdj : float list
module MultipleTesting

from FSharp.Stats.Testing
val benjaminiHochbergFDRBy : projection:('a -> 'b * float) -> rawP:seq<'a> -> ('b * float) list
val x : Contrast
Contrast.Significance: float
Multiple items
module List

from FSharp.Stats

--------------------
module List

from Microsoft.FSharp.Collections

--------------------
type List<'T> =
  | ( [] )
  | ( :: ) of Head: 'T * Tail: 'T list
    interface IReadOnlyList<'T>
    interface IReadOnlyCollection<'T>
    interface IEnumerable
    interface IEnumerable<'T>
    member GetReverseIndex : rank:int * offset:int -> int
    member GetSlice : startIndex:int option * endIndex:int option -> 'T list
    member Head : 'T
    member IsEmpty : bool
    member Item : index:int -> 'T with get
    member Length : int
    ...
val sortBy : projection:('T -> 'Key) -> list:'T list -> 'T list (requires comparison)
Contrast.Index: int
val map : mapping:('T -> 'U) -> list:'T list -> 'U list
val pValAdj : float
val unzip3 : list:('T1 * 'T2 * 'T3) list -> 'T1 list * 'T2 list * 'T3 list
namespace Plotly
namespace Plotly.NET
val myAxis : title:string -> Axis.LinearAxis
val title : string
module Axis

from Plotly.NET
Multiple items
type LinearAxis =
  inherit DynamicObj
  new : unit -> LinearAxis
  static member init : ?AxisType:AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:AutoRange * ?Rangemode:RangeMode * ?Range:Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:TickOptions * ?Mirror:Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:ShowTickOption * ?Showexponent:ShowExponent * ?Exponentformat:ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:AxisAnchorId * ?Side:Side * ?Overlaying:AxisAnchorId * ?Domain:Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> LinearAxis
  static member style : ?AxisType:AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:AutoRange * ?Rangemode:RangeMode * ?Range:Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:TickOptions * ?Mirror:Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:ShowTickOption * ?Showexponent:ShowExponent * ?Exponentformat:ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:AxisAnchorId * ?Side:Side * ?Overlaying:AxisAnchorId * ?Domain:Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> (LinearAxis -> LinearAxis)

--------------------
new : unit -> Axis.LinearAxis
static member Axis.LinearAxis.init : ?AxisType:StyleParam.AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:StyleParam.AutoRange * ?Rangemode:StyleParam.RangeMode * ?Range:StyleParam.Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:StyleParam.TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:StyleParam.TickOptions * ?Mirror:StyleParam.Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:StyleParam.ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:StyleParam.ShowTickOption * ?Showexponent:StyleParam.ShowExponent * ?Exponentformat:StyleParam.ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:StyleParam.AxisAnchorId * ?Side:StyleParam.Side * ?Overlaying:StyleParam.AxisAnchorId * ?Domain:StyleParam.Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> Axis.LinearAxis
module StyleParam

from Plotly.NET
type Mirror =
  | True
  | Ticks
  | False
  | All
  | AllTicks
    static member convert : (Mirror -> obj)
    static member toString : (Mirror -> string)
union case StyleParam.Mirror.All: StyleParam.Mirror
type TickOptions =
  | Outside
  | Inside
  | Empty
    static member convert : (TickOptions -> obj)
    static member toString : (TickOptions -> string)
union case StyleParam.TickOptions.Inside: StyleParam.TickOptions
val myAxisRange : name:string -> float * float -> Axis.LinearAxis
val name : string
val range : float * float
type Range =
  | MinMax of float * float
  | Values of float array
    static member convert : (Range -> obj)
union case StyleParam.Range.MinMax: float * float -> StyleParam.Range
val styleChart : x:string -> y:string -> chart:GenericChart.GenericChart -> GenericChart.GenericChart
val x : string
val y : string
val chart : GenericChart.GenericChart
type Chart =
  static member Area : xy:seq<#IConvertible * #IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Dash:DrawingStyle * ?Width:'a2 -> GenericChart
  static member Area : x:seq<#IConvertible> * y:seq<#IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Dash:DrawingStyle * ?Width:'a2 -> GenericChart
  static member Bar : keysvalues:seq<#IConvertible * #IConvertible> * ?Name:string * ?Showlegend:bool * ?Color:'a2 * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Marker:Marker -> GenericChart
  static member Bar : keys:seq<#IConvertible> * values:seq<#IConvertible> * ?Name:string * ?Showlegend:bool * ?Color:'a2 * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Marker:Marker -> GenericChart
  static member BoxPlot : xy:seq<'a0 * 'a1> * ?Name:string * ?Showlegend:bool * ?Color:string * ?Fillcolor:'a2 * ?Opacity:float * ?Whiskerwidth:'a3 * ?Boxpoints:Boxpoints * ?Boxmean:BoxMean * ?Jitter:'a4 * ?Pointpos:'a5 * ?Orientation:Orientation * ?Marker:Marker * ?Line:Line * ?Alignmentgroup:'a6 * ?Offsetgroup:'a7 * ?Notched:bool * ?NotchWidth:float * ?QuartileMethod:QuartileMethod -> GenericChart
  static member BoxPlot : ?x:'a0 * ?y:'a1 * ?Name:string * ?Showlegend:bool * ?Color:string * ?Fillcolor:'a2 * ?Opacity:float * ?Whiskerwidth:'a3 * ?Boxpoints:Boxpoints * ?Boxmean:BoxMean * ?Jitter:'a4 * ?Pointpos:'a5 * ?Orientation:Orientation * ?Marker:Marker * ?Line:Line * ?Alignmentgroup:'a6 * ?Offsetgroup:'a7 * ?Notched:bool * ?NotchWidth:float * ?QuartileMethod:QuartileMethod -> GenericChart
  static member Bubble : xysizes:seq<#IConvertible * #IConvertible * #IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:Orientation * ?GroupNorm:GroupNorm * ?UseWebGL:bool -> GenericChart
  static member Bubble : x:seq<#IConvertible> * y:seq<#IConvertible> * sizes:seq<#IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:Orientation * ?GroupNorm:GroupNorm * ?UseWebGL:bool -> GenericChart
  static member Candlestick : stockTimeSeries:seq<DateTime * StockData> * ?Increasing:Line * ?Decreasing:Line * ?WhiskerWidth:float * ?Line:Line * ?XCalendar:Calendar -> GenericChart
  static member Candlestick : open:seq<#IConvertible> * high:seq<#IConvertible> * low:seq<#IConvertible> * close:seq<#IConvertible> * x:seq<#IConvertible> * ?Increasing:Line * ?Decreasing:Line * ?WhiskerWidth:float * ?Line:Line * ?XCalendar:Calendar -> GenericChart
  ...
static member Chart.withX_Axis : xAxis:Axis.LinearAxis * ?Id:int -> (GenericChart.GenericChart -> GenericChart.GenericChart)
static member Chart.withY_Axis : yAxis:Axis.LinearAxis * ?Id:int -> (GenericChart.GenericChart -> GenericChart.GenericChart)
val styleChartRange : x:string -> y:string -> float * float -> float * float -> chart:GenericChart.GenericChart -> GenericChart.GenericChart
val rx : float * float
val ry : float * float
val lsdCorrected : GenericChart.GenericChart
val header : string list
val rows : string list list
val sprintf : format:Printf.StringFormat<'T> -> 'T
static member Chart.Table : headerValues:seq<#seq<'a1>> * cellValues:seq<#seq<'a3>> * ?AlignHeader:seq<StyleParam.HorizontalAlign> * ?AlignCells:seq<StyleParam.HorizontalAlign> * ?ColumnWidth:seq<int> * ?ColumnOrder:seq<int> * ?ColorHeader:'a4 * ?ColorCells:'a5 * ?FontHeader:Font * ?FontCells:Font * ?HeightHeader:'a6 * ?HeightCells:'a7 * ?LineHeader:Line * ?LineCells:Line -> GenericChart.GenericChart (requires 'a1 :> System.IConvertible and 'a3 :> System.IConvertible)
Multiple items
type Font =
  inherit DynamicObj
  new : unit -> Font
  static member init : ?Family:FontFamily * ?Size:'a * ?Color:'b * ?Familysrc:'c * ?Sizesrc:'d * ?Colorsrc:'e -> Font
  static member style : ?Family:FontFamily * ?Size:'a0 * ?Color:'a1 * ?Familysrc:'a2 * ?Sizesrc:'a3 * ?Colorsrc:'a4 -> (Font -> Font)

--------------------
new : unit -> Font
static member Font.init : ?Family:StyleParam.FontFamily * ?Size:'a * ?Color:'b * ?Familysrc:'c * ?Sizesrc:'d * ?Colorsrc:'e -> Font
module GenericChart

from Plotly.NET
val toChartHTML : gChart:GenericChart.GenericChart -> string
val hays : contrastMatrix:float [] [] -> data:float [] [] -> Contrast []
val dmg : float [] []
val contrastMatrixDmg : float [] []
val hsdExample : float [] []
val anovaResult : Anova.OneWayAnovaVariationSources
val contrastMatrixHSD : float [] []
val hsdResult : Contrast []
val tukeyHSD : contrastMatrix:float [] [] -> data:float [] [] -> Contrast []
val tukeySignificance : string
Multiple items
module Array

from FSharp.Stats

--------------------
module Array

from Microsoft.FSharp.Collections
val map : mapping:('T -> 'U) -> array:'T [] -> 'U []
Contrast.L: float
Contrast.Statistic: float
val x : string []
val append : array1:'T [] -> array2:'T [] -> 'T []
module String

from Microsoft.FSharp.Core
val concat : sep:string -> strings:seq<string> -> string
val dunnetExample : float [] []
val dunnetContrastMatrix : float [] []
val dunnettResult : CriticalValueContrast []
val dunnetts : contrastMatrix:float [] [] -> data:float [] [] -> criticalTable:Matrix<float> -> CriticalValueContrast []
module Tables

from FSharp.Stats.Testing
val d1 : float list
val d2 : float list
module FisherHotelling

from FSharp.Stats.Testing
val test : dataA:seq<float> -> dataB:seq<float> -> FisherHotelling.HotellingStatistics
val aErrorAcc : GenericChart.GenericChart
val x : float
static member Chart.Line : xy:seq<#System.IConvertible * #System.IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?Dash:'a2 * ?Width:'a3 * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
static member Chart.Line : x:seq<#System.IConvertible> * y:seq<#System.IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?Dash:'a2 * ?Width:'a3 * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
val pValues : float []
val sort : array:'T [] -> 'T [] (requires comparison)
val pValsAdj : (float * float) list
val rev : list:'T list -> 'T list
val bhValues : GenericChart.GenericChart
static member Chart.Combine : gCharts:seq<GenericChart.GenericChart> -> GenericChart.GenericChart
val pi0 : float
module Qvalues

from FSharp.Stats.Testing.MultipleTesting
val pi0Bootstrap : pValues:float [] -> float
val qValues : float []
val ofPValues : pi0:float -> pValues:float [] -> float []
val qValuesRob : float []
val ofPValuesRobust : pi0:float -> pValues:float [] -> float []
val qChart : GenericChart.GenericChart
val qHisto : GenericChart.GenericChart
static member Chart.Histogram : data:seq<#System.IConvertible> * ?Orientation:StyleParam.Orientation * ?Name:string * ?Showlegend:bool * ?Opacity:float * ?Color:string * ?HistNorm:StyleParam.HistNorm * ?HistFunc:StyleParam.HistNorm * ?nBinsx:int * ?nBinsy:int * ?Xbins:Bins * ?Ybins:Bins * ?xError:'a1 * ?yError:'a2 -> GenericChart.GenericChart
Multiple items
type Bins =
  inherit DynamicObj
  new : unit -> Bins
  static member init : ?StartBins:float * ?EndBins:float * ?Size:'a0 -> Bins
  static member style : ?StartBins:float * ?EndBins:float * ?Size:'a0 -> (Bins -> Bins)

--------------------
new : unit -> Bins
static member Bins.init : ?StartBins:float * ?EndBins:float * ?Size:'a0 -> Bins
type HistNorm =
  | None
  | Percent
  | Probability
  | Density
  | ProbabilityDensity
    static member convert : (HistNorm -> obj)
    static member toString : (HistNorm -> string)
union case StyleParam.HistNorm.Density: StyleParam.HistNorm
type DrawingStyle =
  | Solid
  | Dash
  | Dot
  | DashDot
  | User of int
    static member convert : (DrawingStyle -> obj)
    static member toString : (DrawingStyle -> string)
union case StyleParam.DrawingStyle.Dash: StyleParam.DrawingStyle