Cross validation

Binder

Summary: this tutorial demonstrates how to perform several types of cross validation with FSharp.Stats.

Table of contents

Leave-one-out cross validation (LOOCV)

When fitting a data set it often comes down to the selection of the optimal fitting parameter(s). A method to determine these is given by the leave-one-out cross validation (LOOCV). Thereby, the data set is fitted with a given parameter range (smoothing strength, polynomial order etc.) in order to select the best.

Procedure

In each iteration, one data point is excluded from the fitting procedure. The coefficients are determined based on the remaining (n-1) data points. The difference of the excluded point with its corresponding fitted point is measured. In a two-dimensional problem it is the y-intercept of f(xi) and the y_orig at xi.

After every data point was excluded once, the average (squared) distance is calculated and assigned to the corresponding fitting parameter (polynomial order or smoothing strength). The parameter of the model that shows the minimal average error is the best under the given assumptions. It shows the best compromise between over- and underfitting respectively.

Polynomial loocv

let's first create some polynomial fits to cross validate:

open FSharp.Stats
open FSharp.Stats.Fitting
open LinearRegression.OrdinaryLeastSquares

let xV = vector [1. .. 10.]                            
let yV = vector [1.;20.;51.;40.;37.;6.;-10.;-5.;0.;10.]

// the fitting function fits a polynomial of order 'order' to the training data set (xTrain and yTrain) and applies it to xTest
let getFitFuncPolynomial xTrain yTrain (xTest:RowVector<float>) order = 
    let xDat = xTrain |> Matrix.toVector
    let coeffs  = Polynomial.coefficient order xDat yTrain
    let fit     = Polynomial.fit order coeffs (xTest.[0])
    fit

open Plotly.NET

//Some axis styling
let myAxis title = Axis.LinearAxis.init(Title=title,Mirror=StyleParam.Mirror.All,Ticks=StyleParam.TickOptions.Inside,Showgrid=false,Showline=true,Zeroline=false)
let myLogAxis title = Axis.LinearAxis.init(StyleParam.AxisType.Log,Title=title,Mirror=StyleParam.Mirror.All,Ticks=StyleParam.TickOptions.Inside,Showgrid=false,Showline=true,Zeroline=false)
let styleChart x y chart = chart |> Chart.withX_Axis (myAxis x) |> Chart.withY_Axis (myAxis y)

let rawchart() = 
    Chart.Point (xV,yV) 
    |> Chart.withTraceName "raw data"

let chartOrderOpt = 
    [1 .. 2 .. 10]
    |> List.map (fun order -> 
        let coeffs = Polynomial.coefficient order xV yV
        let fit = Polynomial.fit order coeffs
        [1. .. 0.2 .. 10.]
        |> List.map (fun x -> x,fit x)
        |> Chart.Line
        |> Chart.withTraceName (sprintf "order=%i" order)
        )
    |> fun x -> Chart.Combine (rawchart()::x)
    |> Chart.withTitle "polynomial fits"
    |> Chart.withX_Axis (myAxis "x")
    |> Chart.withY_Axis (myAxis "y")

And then crossvalidate across the polynomial orders:

// the error is calculated as the squared difference of fitted and original y value
let error (f1:float) f2 = pown (f1 - f2) 2

/// Leave-one-out cross validation. Returns the mean squared error of each leave-out at the 
/// specific polynomial order. Minimize for model selection.
let loocvPolynomial (xData:Vector<float>) (yData:Vector<float>) order =
    let xDataMat = Matrix.ofVector xData
    let getFitFuncPol xTrain yTrain (xTest:RowVector<float>) = 
        getFitFuncPolynomial xTrain yTrain xTest order
    let meanSquaredError = CrossValidation.loocv xDataMat yData getFitFuncPol error
    
    meanSquaredError

// polynomial orders that should be checked
let ordersToCheck = [|1 .. 10|]

let errorPol = 
    ordersToCheck 
    |> Array.map (fun order -> 
        let error = loocvPolynomial xV yV order
        order,error)

let chartPol = 
    errorPol 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "polynomial order") 
    |> Chart.withY_Axis (myLogAxis "mean error" )
    |> Chart.withTitle "leave one out cross validation (polynomial)"
    
let result = sprintf "The minimal error is obtained by order=%i" (errorPol |> Seq.minBy snd |> fst)
"The minimal error is obtained by order=3"

Smoothing spline loocv

A smoothing spline is a non-parametric fitting procedure, fitting cubic polynomials in each interval given by the basis points.

let's first create some smoothing splines to cross validate:

// the fitting function fits a polynomial of order 'order' to the training data set (xTrain and yTrain) and applies it to xTest
let getFitFuncSpline xDat yDat (xDatTrain: RowVector<float>) lambda =
    let xDatVec = xDat |> Matrix.toVector
    let zippedData = Seq.zip xDatVec yDat |> Array.ofSeq
    let xValTest = xDatTrain.[0]
    Spline.smoothingSpline zippedData (xDat |> Array.ofSeq) lambda xValTest

    /// in loocv the border points are chosen so that the support range of the training data set does not cover the test point.
    /// if splines are used, that are not defined outside the border points use the following:
    //let xDatSupport = Intervals.create (xDatVec |> Seq.min) (xDatVec |> Seq.max)
    //if Intervals.liesInInterval xValTest xDatSupport then 
    //Spline.smoothingSpline zippedData (xDat |> Array.ofSeq) lambda xValTest
    //else nan

let chartSpline = 
    [0.0002;0.002;0.0216;0.2;2.;20.]
    |> List.map (fun lambda -> 
        let fit = Spline.smoothingSpline (Seq.zip xV yV |> Array.ofSeq) (Array.ofSeq xV) lambda
        [1. .. 0.2 .. 10.]
        |> List.map (fun x -> x,fit x)
        |> Chart.Line
        |> Chart.withTraceName (sprintf "l=%.4f" lambda)
        )
    |> fun x -> 
        Chart.Combine (rawchart()::x)
    |> Chart.withX_Axis (myAxis "x") 
    |> Chart.withY_Axis (myAxis "y")
    |> Chart.withTitle "smoothing splines"

And then crossvalidate across different lambda values:

// the error is calculated as the squared difference of fitted and original y value
let errorSpl (f1:float) f2 = 
    /// if xValue is outside of support area of the fitted model (some smoothing spline algorithms), the error should report 0.
    //if nan.Equals f1 then 0.
    //else pown (f1 - f2) 2
    pown (f1 - f2) 2

/// Leave-one-out cross validation. Returns the mean squared error of each leave-out at the 
/// specific regularization parameter (lambda). Minimize the (MSE) for model selection.
let loocvSmoothingSpline (xData:Vector<float>) (yData:Vector<float>) lambda =
    let xDataMat = Matrix.ofVector xData
    let getFitFuncSpl xDat yDat (xDatTrain: RowVector<float>) =
        getFitFuncSpline xDat yDat xDatTrain lambda
    
    CrossValidation.loocv xDataMat yData getFitFuncSpl errorSpl

// smoothing parameter = lambda = regularization parameter
let lambdasToCheck = [|1. .. 15.|] |> Array.map (fun i -> 0.0001 * i**3.)

let errorSpline = 
    lambdasToCheck 
    |> Array.map (fun lambda -> 
        //basisPoints define, where the knots of the spline are located
        let error = loocvSmoothingSpline xV yV lambda
        lambda,error)

let chartSplineError = 
    errorSpline 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "lambda") 
    |> Chart.withY_Axis (myAxis "mean error")
    |> Chart.withTitle "leave one out cross validation (smoothing spline)"
    
let resultSpline = sprintf "The minimal error is obtained by lambda=%f" (errorSpline |> Seq.minBy snd |> fst)
"The minimal error is obtained by lambda=0.021600"

k fold cross validation

The k fold cross validation (kfcv) is a generalized form of the loocv. Rather than excluding every data point separately, kfcv allows the exclusion of data chunks with a defined fraction of the data points. When using k=10, the data is split up into 10 chunks of sub data sets each containing 10% of the data set.

In each loop one chunk is excluded (test data), while the other 9 chunks serve as training data. After 10 (k) loops every single point was evaluated as test data set once and k-1 times as training data. The selection of the subset chunks is random and can be repeated in several iterations. The output contains the average error together with the standardDeviation computed by the given function.

//repeated k fold cross validation for polynomials
let repeatedKFoldPolynomial k (xData: Vector<float>) (yData: Vector<float>) order =
    let xDataMat = xData |> Matrix.Generic.ofVector
    
    let getFitFuncPol xTrain yTrain (xTest:RowVector<float>) = 
        getFitFuncPolynomial xTrain yTrain xTest order
        
    CrossValidation.repeatedKFold k 10 xDataMat yData getFitFuncPol error Seq.stDev

//creates an output for 10 iterations where defined 20 % of the data set are taken as testing data set
let kfPolynomial order = repeatedKFoldPolynomial 5 xV yV order

//repeated k fold cross validation for smoothing splines
let repeatedKFoldSpline k (xData: Vector<float>) (yData: Vector<float>) lambda =
    let xDataMat = xData |> Matrix.ofVector
    
    let getFitFuncSpl xDat yDat (xDatTrain: RowVector<float>) =
        getFitFuncSpline xDat yDat xDatTrain lambda

    CrossValidation.repeatedKFold k 10 xDataMat yData getFitFuncSpl errorSpl Seq.stDev

//creates an output for 10 iterations where defined 20 % of the data set are taken as testing data set
let kfSpline lambda = repeatedKFoldSpline 5 xV yV lambda

The given data set is small and therefore the mean errors show a high variability

let kfp = 
    let errorSplinekf = 
        ordersToCheck 
        |> Array.map (fun order -> 
            //basisPoints define, where the knots of the spline are located
            let error = kfPolynomial order
            (order,error.Error),error.ErrorStDev)
        |> Array.unzip

    fst errorSplinekf 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "order") 
    |> Chart.withY_Axis (myLogAxis "mean error")
    |> Chart.withYErrorStyle (snd errorSplinekf)
    |> Chart.withTitle "kfoldPolynomial error"
let kfs = 
    let errorSplinekf = 
        lambdasToCheck 
        |> Array.map (fun lambda -> 
            //basisPoints define, where the knots of the spline are located
            let error = kfSpline lambda
            (lambda,error.Error),error.ErrorStDev)
        |> Array.unzip

    fst errorSplinekf 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "lambda") 
    |> Chart.withY_Axis (myAxis "mean error")
    |> Chart.withYErrorStyle (snd errorSplinekf)
    |> Chart.withTitle "kfoldSpline error"

Shuffle and split cross validation

The shuffle and split cross validation (sap) is a modified kfcv version. As in kfcv, sap allows the exclusion of data chunks with a defined fraction of the data points. When using p=0.3, 30% of the data are taken as testing data set while 70% serve as training data set. In sap by default only one testing set is evaluated (unlike to kfcv where every data point is once part of a training data set and k-1 times part of testing data set).

Sap can be performed multiple times. Each time the training data fraction is taken randomly from the original data set. Unlike in kfcv overlaps may occur. The output contains the average error together with the standardDeviation computed by the given function.

let shuffleAndSplitPolynomial p iterations (xData: Vector<float>) (yData: Vector<float>) order =
   let xDataMat = xData |> Matrix.ofVector
   
   let getFitFuncPol xTrain yTrain (xTest:RowVector<float>) = 
       getFitFuncPolynomial xTrain yTrain xTest order
   
   CrossValidation.shuffelAndSplit p iterations xDataMat yData getFitFuncPol error Seq.stDev

//creates an output for 5 iterations where random 20 % of the data set are taken as testing data set
let sasPolynomial order = shuffleAndSplitPolynomial 0.2 5 xV yV order

let shuffleAndSplitSpline p iterations (xData: Vector<float>) (yData: Vector<float>) lambda =
    let xDataMat = xData |> Matrix.ofVector
   
    let getFitFuncSpl xDat yDat (xDatTrain: RowVector<float>) =
        getFitFuncSpline xDat yDat xDatTrain lambda
   
    CrossValidation.shuffelAndSplit p iterations xDataMat yData getFitFuncSpl errorSpl Seq.stDev

//creates an output for 5 iterations where random 20 % of the data set are taken as testing data set
let sasSpline lambda = shuffleAndSplitSpline 0.2 5 xV yV lambda

The given data set is small and therefore the mean errors show a high variability.

let sasp = 
    let errorSplinekf = 
        ordersToCheck 
        |> Array.map (fun order -> 
            //basisPoints define, where the knots of the spline are located
            let error = sasPolynomial order
            (order,error.Error),error.ErrorStDev)
        |> Array.unzip

    fst errorSplinekf 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "order") 
    |> Chart.withY_Axis (myLogAxis "mean error")
    |> Chart.withYErrorStyle (snd errorSplinekf)
    |> Chart.withTitle "shuffle_and_split polynomial error"
let sass = 
    let errorSplinekf = 
        lambdasToCheck 
        |> Array.map (fun lambda -> 
            //basisPoints define, where the knots of the spline are located
            let error = sasSpline lambda
            (lambda,error.Error),error.ErrorStDev)
        |> Array.unzip

    fst errorSplinekf 
    |> Chart.Line 
    |> Chart.withX_Axis (myAxis "lambda") 
    |> Chart.withY_Axis (myAxis "mean error")
    |> Chart.withYErrorStyle (snd errorSplinekf)
    |> Chart.withTitle "shuffle_and_split spline error"
Multiple items
namespace FSharp

--------------------
namespace Microsoft.FSharp
namespace FSharp.Stats
namespace FSharp.Stats.Fitting
module LinearRegression

from FSharp.Stats.Fitting
module OrdinaryLeastSquares

from FSharp.Stats.Fitting.LinearRegression
val xV : Vector<float>
Multiple items
val vector : l:seq<float> -> Vector<float>

--------------------
type vector = Vector<float>
val yV : Vector<float>
val getFitFuncPolynomial : xTrain:Matrix<float> -> yTrain:Vector<float> -> xTest:RowVector<float> -> order:int -> float
val xTrain : Matrix<float>
val yTrain : Vector<float>
val xTest : RowVector<float>
Multiple items
module RowVector

from FSharp.Stats

--------------------
type RowVector<'T> =
  interface IEnumerable
  interface IEnumerable<'T>
  interface IStructuralEquatable
  interface IStructuralComparable
  interface IComparable
  new : opsRV:INumeric<'T> option * arrRV:'T array -> RowVector<'T>
  override Equals : yobj:obj -> bool
  override GetHashCode : unit -> int
  member GetSlice : start:int option * finish:int option -> RowVector<'T>
  member Permute : p:permutation -> RowVector<'T>
  ...

--------------------
new : opsRV:INumeric<'T> option * arrRV:'T array -> RowVector<'T>
Multiple items
val float : value:'T -> float (requires member op_Explicit)

--------------------
type float = System.Double

--------------------
type float<'Measure> = float
val order : int
val xDat : vector
Multiple items
module Matrix

from FSharp.Stats

--------------------
type Matrix<'T> =
  | DenseRepr of DenseMatrix<'T>
  | SparseRepr of SparseMatrix<'T>
    interface IEnumerable
    interface IEnumerable<'T>
    interface IStructuralEquatable
    interface IStructuralComparable
    interface IComparable
    override Equals : yobj:obj -> bool
    override GetHashCode : unit -> int
    member GetSlice : start1:int option * finish1:int option * start2:int option * finish2:int option -> Matrix<'T>
    member PermuteColumns : p:permutation -> Matrix<'T>
    member PermuteRows : p:permutation -> Matrix<'T>
    ...
val toVector : x:Matrix<float> -> vector
val coeffs : Vector<float>
module Polynomial

from FSharp.Stats.Fitting.LinearRegression.OrdinaryLeastSquares
val coefficient : order:int -> xData:Vector<float> -> yData:Vector<float> -> Vector<float>
val fit : float
val fit : order:int -> coef:Vector<float> -> x:float -> float
namespace Plotly
namespace Plotly.NET
val myAxis : title:string -> Axis.LinearAxis
val title : string
module Axis

from Plotly.NET
Multiple items
type LinearAxis =
  inherit DynamicObj
  new : unit -> LinearAxis
  static member init : ?AxisType:AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:AutoRange * ?Rangemode:RangeMode * ?Range:Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:TickOptions * ?Mirror:Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:ShowTickOption * ?Showexponent:ShowExponent * ?Exponentformat:ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:AxisAnchorId * ?Side:Side * ?Overlaying:AxisAnchorId * ?Domain:Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> LinearAxis
  static member style : ?AxisType:AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:AutoRange * ?Rangemode:RangeMode * ?Range:Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:TickOptions * ?Mirror:Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:ShowTickOption * ?Showexponent:ShowExponent * ?Exponentformat:ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:AxisAnchorId * ?Side:Side * ?Overlaying:AxisAnchorId * ?Domain:Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> (LinearAxis -> LinearAxis)

--------------------
new : unit -> Axis.LinearAxis
static member Axis.LinearAxis.init : ?AxisType:StyleParam.AxisType * ?Title:string * ?Titlefont:Font * ?Autorange:StyleParam.AutoRange * ?Rangemode:StyleParam.RangeMode * ?Range:StyleParam.Range * ?RangeSlider:RangeSlider * ?Fixedrange:'a * ?Tickmode:StyleParam.TickMode * ?nTicks:'b * ?Tick0:'c * ?dTick:'d * ?Tickvals:'e * ?Ticktext:'f * ?Ticks:StyleParam.TickOptions * ?Mirror:StyleParam.Mirror * ?Ticklen:'g * ?Tickwidth:'h * ?Tickcolor:'i * ?Showticklabels:'j * ?Tickfont:Font * ?Tickangle:'k * ?Tickprefix:'l * ?Showtickprefix:StyleParam.ShowTickOption * ?Ticksuffix:'m * ?Showticksuffix:StyleParam.ShowTickOption * ?Showexponent:StyleParam.ShowExponent * ?Exponentformat:StyleParam.ExponentFormat * ?Tickformat:'n * ?Hoverformat:'o * ?Showline:bool * ?Linecolor:'p * ?Linewidth:'q * ?Showgrid:bool * ?Gridcolor:'r * ?Gridwidth:'s * ?Zeroline:bool * ?Zerolinecolor:'t * ?Zerolinewidth:'a1 * ?Anchor:StyleParam.AxisAnchorId * ?Side:StyleParam.Side * ?Overlaying:StyleParam.AxisAnchorId * ?Domain:StyleParam.Range * ?Position:float * ?IsSubplotObj:'a2 * ?Tickvalssrc:'a3 * ?Ticktextsrc:'a4 * ?Showspikes:'a5 * ?Spikesides:'a6 * ?Spikethickness:'a7 * ?Spikecolor:'a8 * ?Showbackground:'a9 * ?Backgroundcolor:'a10 * ?Showaxeslabels:'a11 -> Axis.LinearAxis
module StyleParam

from Plotly.NET
type Mirror =
  | True
  | Ticks
  | False
  | All
  | AllTicks
    static member convert : (Mirror -> obj)
    static member toString : (Mirror -> string)
union case StyleParam.Mirror.All: StyleParam.Mirror
type TickOptions =
  | Outside
  | Inside
  | Empty
    static member convert : (TickOptions -> obj)
    static member toString : (TickOptions -> string)
union case StyleParam.TickOptions.Inside: StyleParam.TickOptions
val myLogAxis : title:string -> Axis.LinearAxis
type AxisType =
  | Auto
  | Linear
  | Log
  | Date
  | Category
    static member convert : (AxisType -> obj)
    static member toString : (AxisType -> string)
union case StyleParam.AxisType.Log: StyleParam.AxisType
val styleChart : x:string -> y:string -> chart:GenericChart.GenericChart -> GenericChart.GenericChart
val x : string
val y : string
val chart : GenericChart.GenericChart
type Chart =
  static member Area : xy:seq<#IConvertible * #IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Dash:DrawingStyle * ?Width:'a2 -> GenericChart
  static member Area : x:seq<#IConvertible> * y:seq<#IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Dash:DrawingStyle * ?Width:'a2 -> GenericChart
  static member Bar : keysvalues:seq<#IConvertible * #IConvertible> * ?Name:string * ?Showlegend:bool * ?Color:'a2 * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Marker:Marker -> GenericChart
  static member Bar : keys:seq<#IConvertible> * values:seq<#IConvertible> * ?Name:string * ?Showlegend:bool * ?Color:'a2 * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?Marker:Marker -> GenericChart
  static member BoxPlot : xy:seq<'a0 * 'a1> * ?Name:string * ?Showlegend:bool * ?Color:string * ?Fillcolor:'a2 * ?Opacity:float * ?Whiskerwidth:'a3 * ?Boxpoints:Boxpoints * ?Boxmean:BoxMean * ?Jitter:'a4 * ?Pointpos:'a5 * ?Orientation:Orientation * ?Marker:Marker * ?Line:Line * ?Alignmentgroup:'a6 * ?Offsetgroup:'a7 * ?Notched:bool * ?NotchWidth:float * ?QuartileMethod:QuartileMethod -> GenericChart
  static member BoxPlot : ?x:'a0 * ?y:'a1 * ?Name:string * ?Showlegend:bool * ?Color:string * ?Fillcolor:'a2 * ?Opacity:float * ?Whiskerwidth:'a3 * ?Boxpoints:Boxpoints * ?Boxmean:BoxMean * ?Jitter:'a4 * ?Pointpos:'a5 * ?Orientation:Orientation * ?Marker:Marker * ?Line:Line * ?Alignmentgroup:'a6 * ?Offsetgroup:'a7 * ?Notched:bool * ?NotchWidth:float * ?QuartileMethod:QuartileMethod -> GenericChart
  static member Bubble : xysizes:seq<#IConvertible * #IConvertible * #IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:Orientation * ?GroupNorm:GroupNorm * ?UseWebGL:bool -> GenericChart
  static member Bubble : x:seq<#IConvertible> * y:seq<#IConvertible> * sizes:seq<#IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:Orientation * ?GroupNorm:GroupNorm * ?UseWebGL:bool -> GenericChart
  static member Candlestick : stockTimeSeries:seq<DateTime * StockData> * ?Increasing:Line * ?Decreasing:Line * ?WhiskerWidth:float * ?Line:Line * ?XCalendar:Calendar -> GenericChart
  static member Candlestick : open:seq<#IConvertible> * high:seq<#IConvertible> * low:seq<#IConvertible> * close:seq<#IConvertible> * x:seq<#IConvertible> * ?Increasing:Line * ?Decreasing:Line * ?WhiskerWidth:float * ?Line:Line * ?XCalendar:Calendar -> GenericChart
  ...
static member Chart.withX_Axis : xAxis:Axis.LinearAxis * ?Id:int -> (GenericChart.GenericChart -> GenericChart.GenericChart)
static member Chart.withY_Axis : yAxis:Axis.LinearAxis * ?Id:int -> (GenericChart.GenericChart -> GenericChart.GenericChart)
val rawchart : unit -> GenericChart.GenericChart
static member Chart.Point : xy:seq<#System.IConvertible * #System.IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
static member Chart.Point : x:seq<#System.IConvertible> * y:seq<#System.IConvertible> * ?Name:string * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
static member Chart.withTraceName : ?Name:string * ?Showlegend:bool * ?Legendgroup:string * ?Visible:StyleParam.Visible -> (GenericChart.GenericChart -> GenericChart.GenericChart)
val chartOrderOpt : GenericChart.GenericChart
Multiple items
module List

from FSharp.Stats

--------------------
module List

from Microsoft.FSharp.Collections

--------------------
type List<'T> =
  | ( [] )
  | ( :: ) of Head: 'T * Tail: 'T list
    interface IReadOnlyList<'T>
    interface IReadOnlyCollection<'T>
    interface IEnumerable
    interface IEnumerable<'T>
    member GetReverseIndex : rank:int * offset:int -> int
    member GetSlice : startIndex:int option * endIndex:int option -> 'T list
    member Head : 'T
    member IsEmpty : bool
    member Item : index:int -> 'T with get
    member Length : int
    ...
val map : mapping:('T -> 'U) -> list:'T list -> 'U list
val fit : (float -> float)
val x : float
static member Chart.Line : xy:seq<#System.IConvertible * #System.IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?Dash:'c * ?Width:'d * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
static member Chart.Line : x:seq<#System.IConvertible> * y:seq<#System.IConvertible> * ?Name:string * ?ShowMarkers:bool * ?Showlegend:bool * ?MarkerSymbol:StyleParam.Symbol * ?Color:string * ?Opacity:float * ?Labels:seq<string> * ?TextPosition:StyleParam.TextPosition * ?TextFont:Font * ?Dash:'a2 * ?Width:'a3 * ?StackGroup:string * ?Orientation:StyleParam.Orientation * ?GroupNorm:StyleParam.GroupNorm * ?UseWebGL:bool -> GenericChart.GenericChart
val sprintf : format:Printf.StringFormat<'T> -> 'T
val x : GenericChart.GenericChart list
static member Chart.Combine : gCharts:seq<GenericChart.GenericChart> -> GenericChart.GenericChart
static member Chart.withTitle : title:string * ?Titlefont:Font -> (GenericChart.GenericChart -> GenericChart.GenericChart)
module GenericChart

from Plotly.NET
val toChartHTML : gChart:GenericChart.GenericChart -> string
val error : f1:float -> f2:float -> float
val f1 : float
val f2 : float
val pown : x:'T -> n:int -> 'T (requires member get_One and member ( * ) and member ( / ))
val loocvPolynomial : xData:Vector<float> -> yData:Vector<float> -> order:int -> float
 Leave-one-out cross validation. Returns the mean squared error of each leave-out at the
 specific polynomial order. Minimize for model selection.
val xData : Vector<float>
Multiple items
module Vector

from FSharp.Stats

--------------------
type Vector<'T> =
  interface IEnumerable
  interface IEnumerable<'T>
  interface IStructuralEquatable
  interface IStructuralComparable
  interface IComparable
  new : opsV:INumeric<'T> option * arrV:'T array -> Vector<'T>
  override Equals : yobj:obj -> bool
  override GetHashCode : unit -> int
  member GetSlice : start:int option * finish:int option -> Vector<'T>
  member Permute : p:permutation -> Vector<'T>
  ...

--------------------
new : opsV:INumeric<'T> option * arrV:'T array -> Vector<'T>
val yData : Vector<float>
val xDataMat : matrix
val ofVector : x:Vector<float> -> matrix
val getFitFuncPol : (Matrix<float> -> Vector<float> -> RowVector<float> -> float)
val meanSquaredError : float
module CrossValidation

from FSharp.Stats.Fitting
val loocv : xData:Matrix<'T> -> yData:Vector<'T> -> fitFunc:(Matrix<'T> -> Vector<'T> -> RowVector<'T> -> 'T) -> error:('T -> 'T -> 'T) -> 'T (requires member ( + ) and member DivideByInt and member get_Zero)
val ordersToCheck : int []
val errorPol : (int * float) []
Multiple items
module Array

from FSharp.Stats

--------------------
module Array

from Microsoft.FSharp.Collections
val map : mapping:('T -> 'U) -> array:'T [] -> 'U []
val error : float
val chartPol : GenericChart.GenericChart
val result : string
Multiple items
module Seq

from Plotly.NET

--------------------
module Seq

from FSharp.Stats

--------------------
module Seq

from Microsoft.FSharp.Collections
val minBy : projection:('T -> 'U) -> source:seq<'T> -> 'T (requires comparison)
val snd : tuple:('T1 * 'T2) -> 'T2
val fst : tuple:('T1 * 'T2) -> 'T1
val getFitFuncSpline : xDat:Matrix<float> -> yDat:seq<float> -> xDatTrain:RowVector<float> -> lambda:float -> float
val xDat : Matrix<float>
val yDat : seq<float>
val xDatTrain : RowVector<float>
val lambda : float
val xDatVec : vector
val zippedData : (float * float) []
val zip : source1:seq<'T1> -> source2:seq<'T2> -> seq<'T1 * 'T2>
val ofSeq : source:seq<'T> -> 'T []
val xValTest : float
module Spline

from FSharp.Stats.Fitting
val smoothingSpline : data:(float * float) [] -> basispts:float [] -> (float -> float -> float)
val chartSpline : GenericChart.GenericChart
 in loocv the border points are chosen so that the support range of the training data set does not cover the test point.
 if splines are used, that are not defined outside the border points use the following:
val errorSpl : f1:float -> f2:float -> float
val loocvSmoothingSpline : xData:Vector<float> -> yData:Vector<float> -> lambda:float -> float
 if xValue is outside of support area of the fitted model (some smoothing spline algorithms), the error should report 0.
 Leave-one-out cross validation. Returns the mean squared error of each leave-out at the
 specific regularization parameter (lambda). Minimize the (MSE) for model selection.
val getFitFuncSpl : (Matrix<float> -> seq<float> -> RowVector<float> -> float)
val lambdasToCheck : float []
val i : float
val errorSpline : (float * float) []
val chartSplineError : GenericChart.GenericChart
val resultSpline : string
val repeatedKFoldPolynomial : k:int -> xData:Vector<float> -> yData:Vector<float> -> order:int -> CrossValidation.CrossValidationResult<float>
val k : int
val xDataMat : Matrix<float>
module Generic

from FSharp.Stats.MatrixModule
val ofVector : vector:Vector<'a> -> Matrix<'a>
val repeatedKFold : k:int -> iterations:int -> xData:Matrix<'T> -> yData:Vector<'T> -> fit:(Matrix<'T> -> Vector<'T> -> RowVector<'T> -> 'T) -> error:('T -> 'T -> 'T) -> getStDev:(seq<'T> -> 'T) -> CrossValidation.CrossValidationResult<'T> (requires member ( + ) and member DivideByInt and member get_Zero)
val stDev : items:seq<'T> -> 'U (requires member ( - ) and member get_Zero and member DivideByInt and member ( + ) and member ( * ) and member ( + ) and member ( / ) and member Sqrt)
val kfPolynomial : order:int -> CrossValidation.CrossValidationResult<float>
val repeatedKFoldSpline : k:int -> xData:Vector<float> -> yData:Vector<float> -> lambda:float -> CrossValidation.CrossValidationResult<float>
val kfSpline : lambda:float -> CrossValidation.CrossValidationResult<float>
val kfp : GenericChart.GenericChart
val errorSplinekf : (int * float) [] * float []
val error : CrossValidation.CrossValidationResult<float>
CrossValidation.CrossValidationResult.Error: float
CrossValidation.CrossValidationResult.ErrorStDev: float
val unzip : array:('T1 * 'T2) [] -> 'T1 [] * 'T2 []
static member Chart.withYErrorStyle : ?Array:'a * ?Arrayminus:'b * ?Symmetric:'c * ?Color:'d * ?Thickness:'e * ?Width:'f -> (GenericChart.GenericChart -> GenericChart.GenericChart)
val kfs : GenericChart.GenericChart
val errorSplinekf : (float * float) [] * float []
val shuffleAndSplitPolynomial : p:float -> iterations:int -> xData:Vector<float> -> yData:Vector<float> -> order:int -> CrossValidation.CrossValidationResult<float>
val p : float
val iterations : int
val shuffelAndSplit : p:float -> iterations:int -> xData:Matrix<'T> -> yData:Vector<'T> -> fit:(Matrix<'T> -> Vector<'T> -> RowVector<'T> -> 'T) -> error:('T -> 'T -> 'T) -> getStDev:(seq<'T> -> 'T) -> CrossValidation.CrossValidationResult<'T> (requires member ( + ) and member DivideByInt and member get_Zero)
val sasPolynomial : order:int -> CrossValidation.CrossValidationResult<float>
val shuffleAndSplitSpline : p:float -> iterations:int -> xData:Vector<float> -> yData:Vector<float> -> lambda:float -> CrossValidation.CrossValidationResult<float>
val sasSpline : lambda:float -> CrossValidation.CrossValidationResult<float>
val sasp : GenericChart.GenericChart
val sass : GenericChart.GenericChart