Então aqui está o que eu tenho até agora. Parece próximo, mas não tenho certeza de como corrigir os problemas na linha 84 (da segunda à última linha: elif List.append (isolar (a), isolar (b))! = [] E depois List.append (isolar (a), isolar (b))).

    (* val isolate : l:'a list -> 'a list when 'a : equality *)
    let rec isolate (l:'a list) = 
        match l with
        | [] -> []
        | x::xs ->
            if memberof(x,xs)
            then
                let xs = remove (x,l)
                isolate xs
            else isolate xs
    ( * val common : 'a list * 'a list -> 'a list when 'a : equality *)
    let rec common (k: 'a list, l:'a list) = 
        match ((k:'a list),(l:'a list)) with
        | (a, b) -> 
            if a=[] then []
            elif b=[] then []
            elif List.append(isolate(a),isolate(b)) != [] then List.append(isolate(a),isolate(b))
            else []

editar:

solicitado a publicar o código inteiro:

(* val sumlist : l:float list -> float *)
let rec sumlist l =  
    match (l:float list) with
    | [] -> 0.0
    | a::x -> (sumlist x) + a
    (* :: creates a list. *)
sumlist([1.0;2.0;3.0])
(* val squarelist : l:float list -> float list *)
let rec squarelist l = 
    match (l:float list) with
    | [] -> []
    | a::x -> (a*a)::(squarelist x)

(* val mean : l:float list -> float *)
let mean l = 
    match (l:float list) with
    | [] -> 0.0
    | l -> (sumlist l)/(float)l.Length
mean([1.0;2.0;3.0])
(* val mean_diffs : l:float list -> float list *)
let mean_diffs l = 
    match l with
    set a = mean(l)

    | [] -> []
    let rec diffs (a,l)=
        match l with
        | x::xs -> (x-(mean(l))::diffs(xs)
        | [] -> l
mean_diffs([1.0;2.0;3.0])

(* val variance : l:float list -> float *)
let variance l = 
    match (l:float list) with
    | [] -> 0.0
    | l -> (sumlist (squarelist (mean_diffs l)))/(float)l.Length


(* End of question 1 *) (* Do not edit this line. *)

(* Question 2 *) (* Do not edit this line. *)

(* val memberof : 'a * 'a list -> bool when 'a : equality *)
let rec memberof l=
    match (l: 'a * 'a list) with
    | (t,[]) -> false
    | (t, x::xs) when t=x -> true
    | (t, x::xs) -> t=x || memberof(t,xs)

(* val remove : 'a * 'a list -> 'a list when 'a : equality *)
let rec remove ((k:'a),(l:'a list)) = 
    match l with
    | [] -> []
    | x::xs when x=k -> xs
    | x::xs ->x::(remove(k,xs))

(* End of question 2 *) (* Do not edit this line *)

(* Question 3 *) (* Do not edit this line *)

(* val isolate : l:'a list -> 'a list when 'a : equality *)
let rec isolate (l:'a list) = 
    match l with
    | [] -> []
    | x::xs ->
        if memberof(x,xs)
        then
            let xs = remove (x,l)
            isolate xs
        else isolate xs

(* End of question 3 *) (* Do not edit this line *)

(* Question 4 *) (* Do not edit this line *)

(* val common : 'a list * 'a list -> 'a list when 'a : equality *)
let rec common (k: 'a list, l:'a list) = 
    match ((k:'a list),(l:'a list)) with
    | (a, b) -> 
        if a=[] then []
        elif b=[] then []    
        elif List.append(isolate(a),isolate(b)) <> [] then List.append(isolate(a),isolate(b))
        else []
common([1.0;2.0;6.0;10.0],[5.0;6.0;10.0])

Parece que <> corrigiu o problema, mas você tem algum conselho sobre a minha função mean_diffs?