Simulation

[1]:

import scanpy as sc
import os
import pandas as pd
import numpy as np
import pickle as pkl
import matplotlib as mpl
import matplotlib.pyplot as plt
import scipy.stats

sc.settings.verbosity = 3             # verbosity: errors (0), warnings (1), info (2), hints (3)
sc.logging.print_versions()
sc.settings.n_jobs = 6

WARNING: If you miss a compact list, please try `print_header`!
-----
anndata     0.7.4
scanpy      1.6.0
sinfo       0.3.1
-----
PIL                 7.2.0
anndata             0.7.4
backcall            0.2.0
cairo               1.19.1
cffi                1.14.2
colorama            0.4.3
cycler              0.10.0
cython_runtime      NA
dateutil            2.8.1
decorator           4.4.2
get_version         2.1
h5py                2.10.0
igraph              0.8.2
importlib_metadata  1.7.0
ipykernel           5.3.4
ipython_genutils    0.2.0
jedi                0.17.2
joblib              0.16.0
kiwisolver          1.2.0
legacy_api_wrap     1.2
leidenalg           0.8.1
llvmlite            0.33.0+1.g022ab0f
matplotlib          3.3.1
mkl                 2.3.0
mpl_toolkits        NA
natsort             7.0.1
nt                  NA
ntsecuritycon       NA
numba               0.50.1
numexpr             2.7.1
numpy               1.19.1
packaging           20.4
pandas              1.1.1
parso               0.7.0
pickleshare         0.7.5
pkg_resources       NA
prompt_toolkit      3.0.7
pygments            2.6.1
pyparsing           2.4.7
pythoncom           NA
pytz                2020.1
pywintypes          NA
scanpy              1.6.0
scipy               1.5.2
setuptools_scm      NA
sinfo               0.3.1
six                 1.15.0
sklearn             0.23.2
sphinxcontrib       NA
storemagic          NA
tables              3.6.1
texttable           1.6.2
tornado             6.0.4
traitlets           4.3.3
wcwidth             0.2.5
win32api            NA
win32com            NA
win32security       NA
zipp                NA
zmq                 19.0.1
-----
IPython             7.18.1
jupyter_client      6.1.6
jupyter_core        4.6.3
notebook            6.1.1
-----
Python 3.7.7 (default, May  6 2020, 11:45:54) [MSC v.1916 64 bit (AMD64)]
Windows-10-10.0.17763-SP0
12 logical CPU cores, Intel64 Family 6 Model 158 Stepping 10, GenuineIntel
-----
Session information updated at 2020-12-01 10:39
T0 T1 T2 Pro ---> PreA -> A | \-> PreB -> B Kinds of genes: 1. Many markers specific for Pro/PreX/PreY/X/Y: 20 per cell type 2. Many markers shared by X/Y // PreX/PreY: 10 per cell type 3. Few markers ignited at checkpoint of PreX->X and PreY->Y: 10 100 cells for each group TF are ignites near T1.5
[2]:

genes = sum([[t + str(i) for i in range(20)] for t in ['Pro', 'PreX', 'PreY', 'MatureX', 'MatureY']], []) + \
        sum([[t + str(i) for i in range(5)] for t in ['PreX_PreY', 'MatureX_MatureY']], []) + \
        sum([[t + str(i) for i in range(5)] for t in ['Checkpoint']], []) + \
        sum([[t + str(i) for i in range(15)] for t in ['Noninformative_high']], []) + \
        sum([[t + str(i) for i in range(15)] for t in ['Noninformative_low']], []) + \
        sum([[t + str(i) for i in range(15)] for t in ['Noninformative_mid']], []) + \
        sum([[t + str(i) for i in range(10)] for t in ['GradientX', 'GradientY']], [])

[3]:

np.random.seed(0)

[4]:

high = 10.
low = 1.

df = pd.DataFrame(data=low, columns=['Pro', 'PreX', 'PreY', 'MatureX', 'MatureY', 'Checkpoint',
                                     'GradientX', 'GradientY'],
                  index=genes)

df.Pro[df.index.str.contains("Pro")] = high
df.PreX[df.index.str.contains("PreX")] = high
df.PreY[df.index.str.contains("PreY")] = high
df.MatureX[df.index.str.contains("MatureX")] = high
df.MatureY[df.index.str.contains("MatureY")] = high
df.Checkpoint[df.index.str.contains("Checkpoint")] = high
df.GradientX[df.index.str.contains("GradientX")] = high
df.GradientY[df.index.str.contains("GradientY")] = high
df.loc[df.index.str.contains("Noninformative_high"), :] = high
df.loc[df.index.str.contains("Noninformative_mid"), :] = 5.

[5]:

df.style

[5]:
Pro PreX PreY MatureX MatureY Checkpoint GradientX GradientY
Pro0 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro1 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro2 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro3 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro4 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro5 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro6 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro7 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro8 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro9 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro10 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro11 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro12 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro13 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro14 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro15 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro16 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro17 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro18 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Pro19 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX0 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX1 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX2 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX3 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX4 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX5 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX6 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX7 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX8 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX9 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX10 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX11 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX12 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX13 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX14 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX15 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX16 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX17 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX18 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX19 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY0 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY1 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY2 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY3 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY4 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY5 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY6 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY7 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY8 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY9 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY10 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY11 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY12 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY13 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY14 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY15 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY16 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY17 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY18 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreY19 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
MatureX0 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX1 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX2 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX3 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX4 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX5 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX6 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX7 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX8 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX9 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX10 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX11 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX12 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX13 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX14 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX15 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX16 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX17 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX18 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureX19 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000 1.000000
MatureY0 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY1 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY2 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY3 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY4 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY5 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY6 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY7 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY8 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY9 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY10 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY11 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY12 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY13 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY14 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY15 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY16 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY17 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY18 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
MatureY19 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000 1.000000
PreX_PreY0 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX_PreY1 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX_PreY2 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX_PreY3 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
PreX_PreY4 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000 1.000000 1.000000
MatureX_MatureY0 1.000000 1.000000 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000
MatureX_MatureY1 1.000000 1.000000 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000
MatureX_MatureY2 1.000000 1.000000 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000
MatureX_MatureY3 1.000000 1.000000 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000
MatureX_MatureY4 1.000000 1.000000 1.000000 10.000000 10.000000 1.000000 1.000000 1.000000
Checkpoint0 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000
Checkpoint1 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000
Checkpoint2 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000
Checkpoint3 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000
Checkpoint4 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000 1.000000
Noninformative_high0 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high1 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high2 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high3 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high4 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high5 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high6 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high7 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high8 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high9 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high10 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high11 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high12 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high13 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_high14 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000 10.000000
Noninformative_low0 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low2 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low3 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low4 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low5 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low6 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low7 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low8 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low9 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low10 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low11 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low12 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low13 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_low14 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Noninformative_mid0 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid1 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid2 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid3 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid4 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid5 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid6 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid7 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid8 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid9 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid10 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid11 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid12 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid13 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
Noninformative_mid14 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000 5.000000
GradientX0 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX2 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX3 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX4 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX5 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX6 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX7 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX8 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientX9 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 1.000000
GradientY0 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY2 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY3 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY4 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY5 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY6 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY7 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY8 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000
GradientY9 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000 10.000000 
[6]:

X = np.vstack([np.random.uniform(-0.05, 1., [800, 1]), np.random.uniform(1., 2.1, [1200, 1])])
Y = np.vstack([np.random.uniform(-0.05, 1., [800, 1]), np.random.uniform(1., 2.1, [1200, 1])])

[7]:

dispersion = np.vstack([(1. - np.clip(np.maximum(X, 0.) * 1.0, 0., 1.)) * df.Pro.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(X - 1.) * 1.2, 0., 1.)) * df.PreX.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(np.minimum(X, 2.) - 2.), 0., 1.)) * df.MatureX.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(X - 1.5) * 8., 0., 1.)) * df.Checkpoint.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(X - 1.5) * 16., 0., 1.)) * df.Checkpoint.values.reshape([1, -1]) + \
                        (np.clip(np.abs(X * 0.5), 0., 1.)) * df.GradientX.values.reshape([1, -1]),

                        (1. - np.clip(np.maximum(Y, 0.) * 1.0, 0., 1.)) * df.Pro.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(Y - 1.) * 1.2, 0., 1.)) * df.PreY.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(np.minimum(Y, 2.) - 2.), 0., 1.)) * df.MatureY.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(Y - 1.5) * 8., 0., 1.)) * df.Checkpoint.values.reshape([1, -1]) + \
                        (1. - np.clip(np.abs(Y - 1.5) * 16., 0., 1.)) * df.Checkpoint.values.reshape([1, -1]) + \
                        (np.clip(np.abs(Y * 0.5), 0., 1.)) * df.GradientY.values.reshape([1, -1])
                         ])

dispersion.shape

[7]:

(4000, 180)

[8]:

label = []
for i in X:
    label.append('Pro' if i < .5 else 'PreX' if i < 1.5 else 'MatureX')
for i in Y:
    label.append('Pro' if i < .5 else 'PreY' if i < 1.5 else 'MatureY')

[9]:

theta = 10.
expression = np.random.poisson(np.random.gamma(dispersion / np.sum(dispersion, axis=1, keepdims=True) * 10000 / theta, theta))
expression.shape

[9]:

(4000, 180)

[10]:

adata = sc.AnnData(expression)
adata.var_names = df.index
adata.obs['label'] = label

[11]:

adata.obs['time'] = np.clip(X.squeeze().tolist() + Y.squeeze().tolist(), 0., 2.)

[12]:

adata.raw = adata

[13]:

sc.pp.normalize_total(adata, target_sum=1e4)
sc.pp.log1p(adata)
sc.pp.scale(adata, max_value=10)

normalizing counts per cell
    finished ({time_passed})

[14]:

sc.settings.set_figure_params(dpi=80)

[15]:

plt.scatter(adata[:, 'PreX0'].X, adata[:, 'PreY1'].X)

C:\Users\SLiang3\Miniconda3\envs\scanpy37\lib\site-packages\anndata\_core\anndata.py:1094: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  if not is_categorical(df_full[k]):

[15]:

<matplotlib.collections.PathCollection at 0x219ed226188>
_images/simulation_16_2.png 
[118]:

sc.tl.pca(adata, svd_solver='arpack')
sc.pl.pca(adata, color="label", size=10., frameon=False, title="")
sc.pl.pca(adata, color="label", size=10., frameon=False, legend_loc="", title="")

computing PCA
    with n_comps=50
    finished (0:00:00)
_images/simulation_17_1.png
_images/simulation_17_2.png 
[17]:

sc.pl.pca_variance_ratio(adata, log=True)
_images/simulation_18_0.png 
[126]:

sc.pp.neighbors(adata, n_pcs=3)
sc.tl.umap(adata)
adata.obsm['X_umap'][:, 0] = -adata.obsm['X_umap'][:, 0]
adata.obsm['X_umap'][:, 1] = -adata.obsm['X_umap'][:, 1]
sc.pl.umap(adata, color=['label'], legend_loc=None, size=10., frameon=False, title="")

computing neighbors
    using 'X_pca' with n_pcs = 3
    finished: added to `.uns['neighbors']`
    `.obsp['distances']`, distances for each pair of neighbors
    `.obsp['connectivities']`, weighted adjacency matrix (0:00:00)
computing UMAP
    finished: added
    'X_umap', UMAP coordinates (adata.obsm) (0:00:05)
_images/simulation_19_1.png 
[87]:

import sys
sys.path.insert(0,'../marker-selection/')
import scmer

#4000: 62
model = scmer.UmapL1(w=1., lasso=.79e-2, ridge=0., n_pcs=5, perplexity=100., use_beta_in_Q=True,
                     n_threads=6, max_outer_iter=1)
model.fit(adata.X)
adata.var_names[model.w > 0.]

Calculating distance matrix and scaling factors...
Computing pairwise distances...
Using 6 threads...
Mean value of sigma: 0.514251
Done. Elapsed time: 15.72 seconds. Total: 15.72 seconds.
Creating model without batches...
Optimizing using OWLQN (because lasso is nonzero)...
0 loss (before this step): 4.80375337600708 Nonzero (after): 63 Elapsed time: 35.53 seconds. Total: 51.25 seconds.
Final loss: 3.48805570602417 Nonzero: 63 Elapsed time: 0.60 seconds. Total: 51.86 seconds.

[87]:

Index(['Pro0', 'Pro1', 'Pro2', 'Pro3', 'Pro4', 'Pro5', 'Pro6', 'Pro7', 'Pro8',
       'Pro9', 'Pro10', 'Pro11', 'Pro12', 'Pro13', 'Pro14', 'Pro15', 'Pro16',
       'Pro17', 'Pro18', 'Pro19', 'PreX0', 'PreX2', 'PreX4', 'PreX5', 'PreX6',
       'PreX7', 'PreX9', 'PreX11', 'PreX13', 'PreX14', 'PreX16', 'PreX17',
       'PreX18', 'PreY0', 'PreY1', 'PreY4', 'PreY5', 'PreY6', 'PreY8', 'PreY9',
       'PreY11', 'PreY12', 'PreY13', 'PreY14', 'PreY15', 'PreY17', 'PreY18',
       'PreY19', 'PreX_PreY0', 'PreX_PreY1', 'PreX_PreY2', 'PreX_PreY3',
       'PreX_PreY4', 'MatureX_MatureY0', 'MatureX_MatureY1',
       'MatureX_MatureY2', 'MatureX_MatureY3', 'MatureX_MatureY4',
       'Checkpoint4', 'GradientX2', 'GradientX3', 'GradientX8', 'GradientY5'],
      dtype='object')

[88]:

adata.var_names[model.get_mask(45)]

[88]:

Index(['Pro0', 'Pro1', 'Pro2', 'Pro3', 'Pro4', 'Pro5', 'Pro6', 'Pro7', 'Pro8',
       'Pro9', 'Pro10', 'Pro11', 'Pro12', 'Pro13', 'Pro14', 'Pro15', 'Pro16',
       'Pro17', 'Pro18', 'Pro19', 'PreX2', 'PreX5', 'PreX6', 'PreX7', 'PreX11',
       'PreX13', 'PreX14', 'PreY6', 'PreY8', 'PreY15', 'PreY18', 'PreY19',
       'PreX_PreY0', 'PreX_PreY1', 'PreX_PreY2', 'PreX_PreY3', 'PreX_PreY4',
       'MatureX_MatureY0', 'MatureX_MatureY1', 'MatureX_MatureY2',
       'Checkpoint4', 'GradientX2', 'GradientX3', 'GradientX8', 'GradientY5'],
      dtype='object')

[ ]:



[129]:

sc.pl.umap(adata,
           color=['Checkpoint0'],
           ncols=3,
           use_raw=False,
           vmin=-1.5,
           vmax=None,
           legend_loc=None, size=10., color_map='inferno', frameon=False, title="")
_images/simulation_23_0.png 
[136]:

sc.pl.umap(adata,
           color=['MatureX_MatureY0'],
           ncols=3,
           use_raw=False,
           vmin=-1,
           legend_loc=None, size=10., color_map='inferno', frameon=False, title="")
_images/simulation_24_0.png 
[134]:

sc.pl.umap(adata,
           color=['GradientX5'],
           ncols=3,
           use_raw=False,
           vmin=-1,
           legend_loc=None, size=10., color_map='inferno', frameon=False, title="")
_images/simulation_25_0.png 
[135]:

sc.pl.umap(adata,
           color=['Pro0'],
           ncols=3,
           use_raw=False,
           vmin=-0.5,
           legend_loc=None, size=10., color_map='inferno', frameon=False, title="")
_images/simulation_26_0.png

Metrics

[93]:

adata.obs['label_l2'] = adata.obs['label'].astype(str)
mask = np.abs(adata.obs.time - 1.5) * 8. < 0.4
adata.obs['label_l2'][mask] = \
    'checkpoint' + adata.obs['label_l2'][mask].str[-1]

C:\Users\SLiang3\Miniconda3\envs\scanpy37\lib\site-packages\ipykernel_launcher.py:4: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  after removing the cwd from sys.path.

[94]:

print(adata.obs.label.value_counts() / 4000)
print(adata.obs.label_l2.value_counts() / 4000)

PreY       0.24325
PreX       0.22425
Pro        0.21225
MatureX    0.16725
MatureY    0.15300
Name: label, dtype: float64
PreY           0.22925
PreX           0.21225
Pro            0.21225
MatureX        0.15375
MatureY        0.14025
checkpointY    0.02675
checkpointX    0.02550
Name: label_l2, dtype: float64

kNN

[95]:

import sklearn as skl
import scipy as sp
def knn_ri(X, y, n_neighbors=3):
    #y_ = skl.neighbors.KNeighborsClassifier(n_neighbors=n_neighbors).fit(X, y).predict(X)
    pdist = sp.spatial.distance.squareform(sp.spatial.distance.pdist(np.array(X)))
    ind = np.argpartition(pdist, n_neighbors, 1)[:, 1:(n_neighbors + 1)] # smallest one is itself, discard
    y_ = [y[ind[i, :]].sum() > n_neighbors / 2 for i in range(ind.shape[0])]
    return "%.2f / %.2f" % (skl.metrics.precision_score(y_true=y, y_pred=y_), skl.metrics.recall_score(y_true=y, y_pred=y_))
    #return y_

[96]:

from scipy.stats import pearsonr
corr_both = [pearsonr(adata.X[:, i], adata.obs.time.tolist())[0] for i in range(adata.shape[1])]
corr_X = [pearsonr(adata.X[:2000, i], adata.obs.time.tolist()[:2000])[0] for i in range(adata.shape[1])]
corr_Y = [pearsonr(adata.X[2000:, i], adata.obs.time.tolist()[2000:])[0] for i in range(adata.shape[1])]
corr_df = pd.DataFrame(data = {'both': adata.var_names[np.argsort(-np.abs(corr_both))],
                               'X': adata.var_names[np.argsort(-np.abs(corr_X))],
                               'Y': adata.var_names[np.argsort(-np.abs(corr_Y))]})

sc.tl.rank_genes_groups(adata, "label", method='wilcoxon')
deg_df = pd.DataFrame(adata.uns['rank_genes_groups']['names'])

ranking genes

C:\Users\SLiang3\Miniconda3\envs\scanpy37\lib\site-packages\anndata\_core\anndata.py:1192: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  if is_string_dtype(df[key]) and not is_categorical(df[key])
... storing 'label_l2' as categorical

    finished: added to `.uns['rank_genes_groups']`
    'names', sorted np.recarray to be indexed by group ids
    'scores', sorted np.recarray to be indexed by group ids
    'logfoldchanges', sorted np.recarray to be indexed by group ids
    'pvals', sorted np.recarray to be indexed by group ids
    'pvals_adj', sorted np.recarray to be indexed by group ids (0:00:00)

[97]:

n_features = 50
for i in range(corr_df.shape[0]):
    if len(set(corr_df.iloc[:i, ].values.reshape([-1]))) > n_features:
        break
corr_features = list(set(corr_df.iloc[:i, ].values.reshape([-1])))

for i in range(deg_df.shape[0]):
    if len(set(deg_df.iloc[:i, ].values.reshape([-1]))) > n_features:
        break
deg_features = list(set(deg_df.iloc[:i, ].values.reshape([-1])))

[98]:

def calc_precision_df(X, y, adata, model, deg_features, corr_features):
    return pd.DataFrame({label_name:
              {'Original': knn_ri(adata.X, label),
               'SCMER': knn_ri(adata.X[:, model.get_mask(n_features)], label),
               'DEG': knn_ri(adata[:, deg_features].X, label),
               'Correlation': knn_ri(adata[:, corr_features].X, label)
              }
             }
            )

[99]:

label_name = 'checkpointX'
obs_name = 'label_l2'
label = adata.obs[obs_name] == label_name
precision_df = calc_precision_df(X, label, adata, model, deg_features, corr_features)

label_name = 'checkpointY'
obs_name = 'label_l2'
label = adata.obs[obs_name] == label_name
precision_df = precision_df.merge(calc_precision_df(X, label, adata, model, deg_features, corr_features),
                                  left_index=True, right_index=True)

obs_name = 'label'
for label_name in ['Pro', 'PreX', 'PreY', 'MatureX', 'MatureY']:
    label = adata.obs[obs_name] == label_name
    precision_df = precision_df.merge(calc_precision_df(X, label, adata, model, deg_features, corr_features),
                                  left_index=True, right_index=True)

C:\Users\SLiang3\Miniconda3\envs\scanpy37\lib\site-packages\anndata\_core\anndata.py:1094: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  if not is_categorical(df_full[k]):

[100]:

precision_df.loc[['SCMER', 'DEG', 'Correlation']]

[100]:
checkpointX checkpointY Pro PreX PreY MatureX MatureY
SCMER 0.82 / 0.68 0.87 / 0.67 0.97 / 0.96 0.95 / 0.96 0.94 / 0.94 0.95 / 0.96 0.94 / 0.93
DEG 0.61 / 0.34 0.73 / 0.40 0.94 / 0.95 0.94 / 0.93 0.95 / 0.94 0.94 / 0.95 0.95 / 0.96
Correlation 0.48 / 0.36 0.43 / 0.28 0.91 / 0.96 0.76 / 0.67 0.76 / 0.67 0.88 / 0.95 0.88 / 0.92 
[33]:

corr_features

[33]:

['MatureX19',
 'MatureX9',
 'Pro7',
 'Pro10',
 'Pro18',
 'MatureY2',
 'Pro1',
 'MatureY16',
 'MatureX5',
 'Pro0',
 'MatureX0',
 'MatureX18',
 'MatureX16',
 'MatureY14',
 'Pro12',
 'MatureY19',
 'MatureX11',
 'MatureX_MatureY4',
 'MatureX12',
 'MatureY4',
 'Pro11',
 'Pro3',
 'MatureY5',
 'Pro2',
 'MatureX_MatureY1',
 'Pro4',
 'MatureX4',
 'Pro9',
 'MatureX8',
 'MatureY8',
 'Pro16',
 'MatureY0',
 'MatureY10',
 'MatureX2',
 'MatureY11',
 'Pro19',
 'Pro5',
 'Pro13',
 'Pro14',
 'Pro17',
 'MatureX_MatureY2',
 'MatureY7',
 'Pro6',
 'Pro15',
 'MatureY13',
 'MatureX_MatureY3',
 'MatureY12',
 'MatureX14',
 'Pro8',
 'MatureY18',
 'MatureX_MatureY0',
 'MatureX17']

[34]:

deg_features

[34]:

['PreY7',
 'PreY9',
 'Pro7',
 'MatureX6',
 'PreX0',
 'Pro10',
 'MatureX9',
 'MatureX14',
 'MatureY16',
 'PreX15',
 'MatureX5',
 'PreY5',
 'PreY10',
 'MatureX0',
 'MatureX16',
 'MatureY14',
 'MatureY19',
 'MatureX11',
 'PreX12',
 'PreX10',
 'MatureX7',
 'PreX19',
 'MatureY17',
 'PreY19',
 'Pro11',
 'MatureY5',
 'Pro3',
 'Pro4',
 'MatureY6',
 'PreX16',
 'PreX11',
 'PreX14',
 'MatureY0',
 'MatureY10',
 'MatureX2',
 'PreY1',
 'PreY14',
 'Pro13',
 'Pro5',
 'PreY16',
 'PreX2',
 'Pro14',
 'PreX6',
 'PreX4',
 'Pro6',
 'Pro15',
 'MatureY3',
 'MatureX13',
 'MatureY12',
 'PreY4',
 'Pro12',
 'MatureY18',
 'PreY3',
 'PreY6',
 'MatureX17']

[ ]:



[35]:

deg_df.style

[35]:
MatureX MatureY PreX PreY Pro
0 MatureX14 MatureY0 PreX0 PreY5 Pro12
1 MatureX17 MatureY16 PreX2 PreY16 Pro10
2 MatureX5 MatureY12 PreX11 PreY9 Pro7
3 MatureX16 MatureY14 PreX19 PreY6 Pro6
4 MatureX6 MatureY10 PreX15 PreY7 Pro14
5 MatureX9 MatureY6 PreX10 PreY1 Pro13
6 MatureX11 MatureY5 PreX16 PreY10 Pro15
7 MatureX0 MatureY19 PreX12 PreY14 Pro4
8 MatureX13 MatureY17 PreX4 PreY19 Pro11
9 MatureX7 MatureY3 PreX14 PreY3 Pro3
10 MatureX2 MatureY18 PreX6 PreY4 Pro5
11 MatureX4 MatureY15 PreX5 PreY8 Pro1
12 MatureX3 MatureY2 PreX3 PreY11 Pro0
13 MatureX15 MatureY1 PreX1 PreY18 Pro17
14 MatureX18 MatureY8 PreX18 PreY2 Pro19
15 MatureX19 MatureY13 PreX17 PreY0 Pro8
16 MatureX10 MatureY9 PreX8 PreY13 Pro16
17 MatureX8 MatureY7 PreX13 PreY17 Pro18
18 MatureX12 MatureY4 PreX7 PreY15 Pro2
19 MatureX1 MatureY11 PreX9 PreY12 Pro9
20 GradientX5 GradientY6 GradientX9 GradientY9 Noninformative_low7
21 GradientX4 GradientY5 GradientX2 GradientY7 Noninformative_low5
22 GradientX6 GradientY2 GradientX3 GradientY3 Noninformative_mid12
23 GradientX8 GradientY4 GradientX8 GradientY0 Noninformative_low14
24 GradientX3 GradientY1 GradientX7 GradientY4 Noninformative_mid10
25 GradientX7 GradientY8 GradientX1 GradientY5 Noninformative_low2
26 GradientX1 GradientY0 GradientX0 GradientY1 Noninformative_low11
27 GradientX0 GradientY3 PreX_PreY3 GradientY2 Noninformative_mid13
28 GradientX2 GradientY7 PreX_PreY0 GradientY8 Noninformative_mid2
29 GradientX9 GradientY9 GradientX5 GradientY6 Noninformative_high9
30 MatureX_MatureY0 MatureX_MatureY2 GradientX6 PreX_PreY1 Noninformative_mid9
31 MatureX_MatureY1 MatureX_MatureY0 GradientX4 PreX_PreY2 Noninformative_low4
32 MatureX_MatureY4 MatureX_MatureY4 PreX_PreY1 PreX_PreY4 Noninformative_low13
33 MatureX_MatureY3 MatureX_MatureY1 PreX_PreY4 PreX_PreY3 Noninformative_high8
34 MatureX_MatureY2 MatureX_MatureY3 PreX_PreY2 PreX_PreY0 Noninformative_low8
35 Noninformative_high6 Noninformative_high11 MatureX2 MatureY13 Noninformative_low0
36 Noninformative_high4 Noninformative_high3 MatureX12 MatureY11 Noninformative_mid6
37 Noninformative_high5 Noninformative_high0 MatureX5 MatureY9 Noninformative_high6
38 Noninformative_high13 Checkpoint1 MatureX16 MatureY4 Noninformative_low3
39 Noninformative_high12 Checkpoint0 MatureX0 MatureY10 Noninformative_mid4
40 Noninformative_mid14 Checkpoint4 MatureX4 MatureY19 Noninformative_high14
41 Checkpoint3 Checkpoint2 MatureX9 MatureY8 Noninformative_low1
42 Noninformative_high2 Noninformative_mid6 MatureX14 MatureY7 Noninformative_mid7
43 Noninformative_mid0 Noninformative_mid8 MatureX6 MatureY6 Noninformative_high1
44 Noninformative_high7 Noninformative_mid9 MatureX13 MatureY18 Noninformative_mid5
45 Noninformative_high11 Noninformative_high1 MatureX7 MatureY16 Noninformative_high13
46 Noninformative_high3 Noninformative_mid11 MatureX18 MatureY17 Noninformative_high7
47 Checkpoint4 Noninformative_high12 MatureX10 MatureY5 Noninformative_low6
48 Checkpoint0 Noninformative_low12 MatureX15 MatureY14 Noninformative_high2
49 Noninformative_mid7 Noninformative_mid4 MatureX17 MatureY12 Noninformative_low10
50 Checkpoint1 Noninformative_mid3 MatureX1 MatureY1 Noninformative_mid11
51 Noninformative_high1 Noninformative_low14 MatureX3 MatureY0 Noninformative_high10
52 Noninformative_high14 Noninformative_mid1 MatureX11 MatureY2 Noninformative_mid14
53 Noninformative_high8 Noninformative_high10 MatureX19 MatureY3 Noninformative_high5
54 Noninformative_low8 Noninformative_low9 MatureX8 MatureY15 Noninformative_high4
55 Checkpoint2 Noninformative_low2 Checkpoint3 Noninformative_low1 Noninformative_mid1
56 Noninformative_mid6 Noninformative_high9 Noninformative_low3 Noninformative_mid1 Noninformative_mid8
57 Noninformative_low0 Noninformative_mid12 Checkpoint4 Noninformative_low11 Noninformative_low9
58 Noninformative_mid3 Noninformative_low8 Noninformative_mid2 Checkpoint0 Noninformative_high11
59 Noninformative_mid13 Noninformative_mid0 Noninformative_low12 Noninformative_low6 Noninformative_high0
60 Noninformative_mid8 Noninformative_high7 Checkpoint2 Noninformative_mid3 Noninformative_low12
61 Noninformative_mid10 Noninformative_high8 Noninformative_high2 Noninformative_low10 Noninformative_high12
62 Noninformative_low4 Noninformative_mid13 Noninformative_low9 Noninformative_mid5 Noninformative_mid0
63 Noninformative_low13 Noninformative_mid5 Checkpoint1 Noninformative_low13 Noninformative_high3
64 Noninformative_high10 Noninformative_mid14 Noninformative_low4 Noninformative_mid4 Noninformative_mid3
65 Noninformative_high9 Noninformative_mid7 Noninformative_high1 Checkpoint2 Checkpoint3
66 Noninformative_low6 Noninformative_low3 Noninformative_mid5 Noninformative_mid0 Checkpoint1
67 Noninformative_low11 Noninformative_high13 Noninformative_mid3 Noninformative_high10 Checkpoint2
68 Noninformative_mid9 Noninformative_high14 Noninformative_low10 Noninformative_low9 PreY15
69 Noninformative_low10 Noninformative_low0 Noninformative_low6 Noninformative_low7 Checkpoint4
70 Noninformative_high0 Noninformative_high6 Noninformative_low0 Noninformative_low12 Checkpoint0
71 Noninformative_low2 Noninformative_high5 Checkpoint0 Noninformative_mid11 PreX19
72 Noninformative_mid12 Noninformative_high2 Noninformative_mid8 Noninformative_low5 PreX15
73 Noninformative_mid2 Checkpoint3 Noninformative_high13 Noninformative_mid10 PreX7
74 Noninformative_mid1 Noninformative_low10 Noninformative_low14 Noninformative_mid9 PreX9
75 Noninformative_low5 Noninformative_high4 Noninformative_low5 Checkpoint1 PreY13
76 Noninformative_mid11 Noninformative_low7 Noninformative_mid11 Noninformative_high0 PreX6
77 Noninformative_low3 Noninformative_low4 Noninformative_high4 Noninformative_high8 PreX18
78 Noninformative_low1 Noninformative_low1 Noninformative_low2 Noninformative_mid12 PreX4
79 Noninformative_mid4 Noninformative_mid10 Noninformative_low13 Checkpoint4 PreX1
80 Noninformative_low9 Noninformative_low6 Noninformative_low7 Noninformative_high14 PreY4
81 Noninformative_low7 Noninformative_low11 Noninformative_mid14 Noninformative_low8 PreY8
82 Noninformative_mid5 Noninformative_low13 Noninformative_high9 Checkpoint3 PreX13
83 Noninformative_low12 Noninformative_low5 Noninformative_low1 Noninformative_low14 PreX3
84 Noninformative_low14 Noninformative_mid2 Noninformative_high5 Noninformative_high5 PreX8
85 PreX10 PreY12 Noninformative_mid4 Noninformative_mid2 PreY18
86 PreX12 PreY17 Noninformative_mid7 Noninformative_mid7 PreX0
87 PreX2 PreY5 Noninformative_high3 Noninformative_high12 PreY1
88 PreX8 PreY8 Noninformative_high0 Noninformative_high7 PreY6
89 PreX17 PreY9 Noninformative_high7 Noninformative_high3 PreX16
90 PreX0 PreY0 Noninformative_mid1 Noninformative_low4 PreY5
91 PreX4 PreY16 Noninformative_high11 Noninformative_high9 PreX14
92 PreX11 PreY18 Noninformative_mid6 Noninformative_mid8 PreY3
93 PreX6 PreY2 Noninformative_high10 Noninformative_mid13 PreY0
94 PreX19 PreY19 Noninformative_mid0 Noninformative_mid14 PreX12
95 PreX14 PreY10 Noninformative_mid10 Noninformative_low0 PreX5
96 PreX5 PreY11 Noninformative_high14 Noninformative_mid6 PreY10
97 PreX3 PreY7 Noninformative_high12 Noninformative_high4 PreY11
98 PreX13 PreY3 Noninformative_mid13 Noninformative_low2 PreX2
99 PreX15 PreY14 Noninformative_low11 Noninformative_high2 PreX17
100 PreX18 PreY15 Noninformative_mid12 Noninformative_high6 PreX11
101 PreX7 PreY1 Noninformative_low8 Noninformative_low3 PreY7
102 PreX9 PreY4 Noninformative_high6 Noninformative_high13 PreY17
103 PreX16 PreY13 Noninformative_mid9 Noninformative_high11 PreY14
104 PreX1 PreY6 Noninformative_high8 Noninformative_high1 PreY12
105 MatureY15 MatureX8 Pro4 Pro1 PreY2
106 MatureY16 MatureX1 Pro15 Pro8 PreY19
107 MatureY2 MatureX7 Pro17 Pro9 PreX10
108 MatureY3 MatureX11 Pro19 Pro3 PreY9
109 MatureY19 MatureX19 Pro14 Pro11 PreY16
110 MatureY13 MatureX9 Pro2 Pro17 MatureX0
111 MatureY18 MatureX10 Pro10 Pro12 GradientY6
112 MatureY14 MatureX3 Pro12 Pro18 GradientY1
113 MatureY5 MatureX17 Pro18 Pro14 GradientY9
114 MatureY6 MatureX5 Pro5 Pro2 GradientX4
115 MatureY9 MatureX15 Pro16 Pro16 MatureX7
116 MatureY17 MatureX4 Pro7 Pro5 GradientX2
117 MatureY12 MatureX14 Pro6 Pro13 MatureX15
118 MatureY0 MatureX6 Pro8 Pro4 GradientY8
119 MatureY7 MatureX13 Pro11 Pro0 GradientX0
120 MatureY1 MatureX16 Pro1 Pro7 GradientX7
121 MatureY4 MatureX12 Pro13 Pro10 GradientX1
122 MatureY8 MatureX2 Pro0 Pro15 MatureY1
123 MatureY10 MatureX0 Pro3 Pro6 GradientY2
124 MatureY11 MatureX18 Pro9 Pro19 MatureX13
125 PreY9 PreX16 MatureX_MatureY1 MatureX_MatureY3 MatureX14
126 PreY15 PreX1 MatureX_MatureY4 MatureX_MatureY4 GradientX8
127 PreY4 PreX13 MatureX_MatureY3 MatureX_MatureY0 MatureY12
128 PreY13 Pro0 MatureX_MatureY0 MatureX_MatureY1 MatureY3
129 PreY10 PreX6 MatureX_MatureY2 MatureX_MatureY2 GradientX9
130 PreY2 PreX15 MatureY4 MatureX18 MatureX1
131 PreY17 PreX17 MatureY10 MatureX19 MatureY13
132 PreY18 PreX14 MatureY0 MatureX6 MatureX3
133 PreY11 PreX0 MatureY11 MatureX9 MatureY8
134 PreY6 PreX5 MatureY18 MatureX16 MatureX10
135 PreY12 PreX4 MatureY16 MatureX2 MatureY11
136 PreY14 PreX12 MatureY2 MatureX3 GradientY4
137 PreY16 PreX8 MatureY15 MatureX4 MatureY15
138 PreY19 PreX2 MatureY6 MatureX11 MatureY14
139 PreY7 PreX10 MatureY5 MatureX8 MatureY9
140 PreY5 Pro6 MatureY3 MatureX13 MatureX18
141 Pro13 PreX11 MatureY7 MatureX12 MatureX11
142 Pro15 PreX3 MatureY17 MatureX17 MatureY7
143 PreY1 PreX18 MatureY8 MatureX10 MatureY2
144 Pro7 PreX9 MatureY1 MatureX14 MatureX8
145 PreY3 Pro5 MatureY12 MatureX1 MatureX4
146 Pro19 PreX7 MatureY14 MatureX0 MatureX17
147 Pro3 PreX19 MatureY9 MatureX15 MatureX19
148 Pro18 Pro16 MatureY19 MatureX5 MatureY17
149 Pro6 Pro13 MatureY13 MatureX7 MatureY19
150 Pro10 Pro7 PreY19 PreX11 GradientX6
151 Pro2 Pro19 PreY16 PreX9 GradientX3
152 Pro9 Pro9 PreY14 PreX3 GradientY5
153 Pro14 Pro10 PreY7 PreX7 GradientY7
154 Pro11 Pro3 PreY6 PreX10 MatureX6
155 Pro16 Pro8 PreY0 PreX18 GradientX5
156 Pro4 Pro2 PreY3 PreX1 MatureX16
157 Pro17 Pro11 PreY1 PreX5 MatureX5
158 Pro12 Pro1 PreY2 PreX17 MatureY5
159 PreY8 Pro18 PreY13 PreX14 GradientY3
160 PreY0 Pro4 PreY8 PreX13 MatureY0
161 Pro1 Pro15 PreY11 PreX2 GradientY0
162 Pro5 Pro17 PreY12 PreX8 MatureX12
163 Pro8 Pro14 PreY17 PreX16 MatureX2
164 Pro0 Pro12 PreY9 PreX19 MatureX9
165 GradientY7 GradientX9 PreY4 PreX4 MatureY6
166 GradientY5 GradientX7 PreY10 PreX12 MatureY10
167 GradientY3 GradientX8 PreY18 PreX0 MatureY18
168 GradientY0 GradientX0 PreY5 PreX6 MatureY16
169 GradientY2 GradientX6 PreY15 PreX15 MatureY4
170 GradientY6 GradientX5 GradientY8 GradientX6 PreX_PreY2
171 GradientY1 GradientX3 GradientY4 GradientX5 PreX_PreY0
172 GradientY9 GradientX2 GradientY0 GradientX4 PreX_PreY4
173 GradientY4 GradientX1 GradientY5 GradientX3 PreX_PreY3
174 GradientY8 GradientX4 GradientY3 GradientX1 PreX_PreY1
175 PreX_PreY0 PreX_PreY2 GradientY2 GradientX0 MatureX_MatureY2
176 PreX_PreY1 PreX_PreY1 GradientY9 GradientX2 MatureX_MatureY0
177 PreX_PreY4 PreX_PreY4 GradientY7 GradientX7 MatureX_MatureY1
178 PreX_PreY3 PreX_PreY3 GradientY6 GradientX8 MatureX_MatureY3
179 PreX_PreY2 PreX_PreY0 GradientY1 GradientX9 MatureX_MatureY4 
[36]:

len(set(deg_df.iloc[:20, ].values.reshape([-1])))

[36]:

100

[37]:

corr_df.style

[37]:
both X Y
0 Pro12 MatureX2 MatureY5
1 Pro4 MatureX4 MatureY16
2 Pro1 MatureX19 MatureY18
3 MatureX_MatureY3 MatureX5 MatureY14
4 MatureX_MatureY4 MatureX17 MatureY10
5 Pro14 MatureX14 Pro1
6 Pro2 MatureX18 Pro4
7 Pro3 MatureX12 MatureY11
8 MatureX_MatureY0 MatureX16 Pro12
9 Pro5 MatureX11 MatureY2
10 Pro16 MatureX_MatureY3 MatureX_MatureY4
11 Pro18 Pro12 MatureY0
12 MatureX_MatureY1 Pro17 MatureY12
13 Pro17 Pro5 Pro3
14 Pro10 Pro2 MatureY7
15 Pro11 Pro6 Pro14
16 Pro8 MatureX0 MatureX_MatureY3
17 Pro15 Pro4 MatureY19
18 Pro6 MatureX9 MatureY4
19 Pro9 Pro14 Pro13
20 MatureX_MatureY2 MatureX_MatureY4 Pro11
21 Pro7 MatureX8 Pro18
22 Pro13 MatureX_MatureY0 MatureX_MatureY1
23 Pro19 Pro1 MatureY8
24 Pro0 Pro15 MatureY13
25 MatureX5 MatureX13 MatureY6
26 MatureX8 Pro10 MatureY15
27 MatureX9 Pro8 Pro9
28 MatureX10 Pro7 Pro16
29 MatureX1 MatureX7 MatureY3
30 MatureX19 Pro0 MatureY1
31 MatureX4 Pro16 Pro2
32 MatureX17 Pro3 MatureY17
33 MatureX2 Pro18 Pro10
34 MatureX16 MatureX_MatureY1 MatureX_MatureY0
35 MatureY18 MatureX6 MatureY9
36 MatureX6 MatureX3 Pro5
37 MatureX12 MatureX10 MatureX_MatureY2
38 MatureX3 Pro11 Pro19
39 MatureX11 MatureX15 Pro8
40 MatureX14 MatureX_MatureY2 Pro15
41 MatureY16 Pro19 Pro17
42 MatureX15 MatureX1 Pro7
43 MatureX7 Pro9 Pro6
44 MatureY3 Pro13 Pro0
45 MatureX13 GradientX7 GradientY5
46 MatureY10 GradientX5 GradientY8
47 MatureY6 GradientX3 GradientY1
48 MatureY2 GradientX8 GradientY3
49 MatureX18 GradientX4 GradientY6
50 MatureY17 GradientX9 GradientY2
51 MatureY5 GradientX6 GradientY0
52 MatureY12 GradientX0 GradientY4
53 MatureY4 GradientX2 GradientY7
54 MatureY13 GradientX1 GradientY9
55 MatureY14 PreX9 PreY6
56 MatureX0 PreX7 PreY13
57 MatureY0 PreX_PreY3 PreY15
58 MatureY19 PreX_PreY0 PreY4
59 MatureY15 PreX3 PreX_PreY0
60 MatureY7 PreX_PreY2 PreY11
61 MatureY9 PreX18 PreY3
62 MatureY1 PreX_PreY4 PreX_PreY2
63 MatureY11 PreX19 PreY8
64 MatureY8 PreX1 PreY1
65 GradientX6 PreX13 PreY14
66 GradientX9 PreX15 PreX_PreY4
67 GradientX5 PreX16 PreY0
68 GradientX7 PreX17 PreX_PreY1
69 GradientX3 PreX11 PreY18
70 GradientX0 PreX5 PreY10
71 GradientX8 PreX_PreY1 PreY7
72 GradientX2 PreX14 PreY19
73 GradientX1 PreX0 PreY5
74 GradientY5 PreX6 PreY2
75 GradientX4 PreX8 PreY17
76 GradientY7 PreX4 PreX_PreY3
77 GradientY3 PreX2 Checkpoint0
78 GradientY0 PreX12 PreY12
79 PreX_PreY0 Checkpoint3 PreY9
80 PreX_PreY2 PreX10 PreY16
81 GradientY2 Checkpoint1 Checkpoint1
82 PreX_PreY3 Checkpoint4 Checkpoint4
83 GradientY6 Noninformative_high6 Checkpoint2
84 PreX_PreY4 Noninformative_high13 Noninformative_high3
85 GradientY1 Checkpoint0 Noninformative_low12
86 GradientY8 Checkpoint2 Noninformative_high0
87 GradientY4 Noninformative_high4 Noninformative_high11
88 PreX_PreY1 Noninformative_high3 Noninformative_high12
89 GradientY9 Noninformative_mid3 Checkpoint3
90 Checkpoint0 Noninformative_high11 Noninformative_mid6
91 Checkpoint1 Noninformative_mid14 Noninformative_mid8
92 Checkpoint4 Noninformative_high5 Noninformative_mid0
93 Checkpoint2 Noninformative_high12 Noninformative_mid1
94 Checkpoint3 Noninformative_high0 Noninformative_high1
95 PreY1 Noninformative_high14 MatureX9
96 Noninformative_high3 Noninformative_high7 Noninformative_low9
97 PreY15 Noninformative_mid0 Noninformative_mid3
98 PreY6 Noninformative_mid7 Noninformative_mid5
99 Noninformative_high11 Noninformative_high1 Noninformative_mid9
100 PreY13 Noninformative_high2 Noninformative_high9
101 PreX7 Noninformative_mid8 Noninformative_mid14
102 PreY0 Noninformative_mid6 GradientX9
103 PreY8 MatureY13 Noninformative_mid11
104 Noninformative_high0 Noninformative_high8 MatureX8
105 Noninformative_high12 MatureY16 PreX17
106 PreX9 Noninformative_low4 Noninformative_high10
107 Noninformative_mid3 MatureY18 Noninformative_high8
108 PreX19 MatureY6 MatureX10
109 PreX18 Noninformative_mid11 MatureX1
110 PreY3 GradientY7 GradientX6
111 Noninformative_mid14 Noninformative_high10 MatureX5
112 PreY5 MatureY3 Noninformative_high7
113 Noninformative_high13 GradientY5 Noninformative_mid7
114 PreY4 MatureY17 Noninformative_high14
115 PreX3 PreY11 Noninformative_high2
116 PreY7 Noninformative_high9 Noninformative_mid4
117 PreX15 Noninformative_low0 PreX10
118 PreY18 PreY9 Noninformative_mid13
119 PreX13 Noninformative_mid10 Noninformative_high5
120 PreX1 GradientY0 GradientX5
121 PreY11 PreY4 GradientX7
122 Noninformative_mid0 Noninformative_low8 Noninformative_low10
123 Noninformative_mid8 PreY2 GradientX2
124 PreY14 MatureY12 MatureX6
125 PreX14 GradientY3 Noninformative_high13
126 Noninformative_high5 Noninformative_low2 Noninformative_mid12
127 PreX6 PreY14 Noninformative_low1
128 Noninformative_high6 PreY17 Noninformative_low6
129 Noninformative_high4 PreY10 Noninformative_low3
130 Noninformative_mid6 MatureY1 PreX5
131 PreY19 PreY19 PreX16
132 PreY10 Noninformative_low10 GradientX0
133 Noninformative_high1 Noninformative_low13 PreX9
134 PreX16 MatureY19 PreX0
135 PreX11 MatureY4 Noninformative_low2
136 PreY2 Noninformative_mid1 MatureX16
137 PreX5 MatureY10 Noninformative_low8
138 PreY16 Noninformative_low6 Noninformative_low0
139 PreX2 PreY1 PreX1
140 PreX4 Noninformative_low3 PreX8
141 PreX0 PreY12 PreX11
142 Noninformative_high7 PreY6 Noninformative_high4
143 Noninformative_high14 Noninformative_low11 PreX13
144 PreY17 Noninformative_mid4 MatureX14
145 PreX8 Noninformative_mid13 MatureX19
146 Noninformative_mid7 Noninformative_mid2 GradientX8
147 PreY12 PreY13 PreX3
148 Noninformative_high2 Noninformative_low12 PreX12
149 Noninformative_low12 GradientY2 PreX7
150 Noninformative_high8 PreY18 MatureX3
151 Noninformative_mid11 MatureY14 PreX14
152 Noninformative_high9 MatureY15 Noninformative_low14
153 Noninformative_high10 Noninformative_mid9 Noninformative_low13
154 PreX17 MatureY2 Noninformative_low7
155 PreX12 PreY7 GradientX3
156 Noninformative_mid1 MatureY0 MatureX13
157 PreY9 Noninformative_low9 PreX4
158 Noninformative_mid9 PreY3 GradientX1
159 PreX10 MatureY5 MatureX18
160 Noninformative_low9 MatureY9 PreX18
161 Noninformative_low0 MatureY7 MatureX0
162 Noninformative_low10 PreY15 MatureX4
163 Noninformative_mid4 PreY16 MatureX15
164 Noninformative_low8 Noninformative_low7 PreX15
165 Noninformative_mid13 Noninformative_low1 Noninformative_low4
166 Noninformative_low2 MatureY11 Noninformative_high6
167 Noninformative_low4 GradientY6 PreX6
168 Noninformative_mid5 GradientY4 PreX2
169 Noninformative_low6 Noninformative_mid5 MatureX2
170 Noninformative_low3 GradientY1 MatureX12
171 Noninformative_low13 Noninformative_low14 Noninformative_mid10
172 Noninformative_mid10 GradientY9 MatureX11
173 Noninformative_low1 MatureY8 MatureX17
174 Noninformative_mid12 Noninformative_mid12 MatureX7
175 Noninformative_mid2 PreY0 Noninformative_low5
176 Noninformative_low14 GradientY8 Noninformative_mid2
177 Noninformative_low11 Noninformative_low5 GradientX4
178 Noninformative_low5 PreY8 PreX19
179 Noninformative_low7 PreY5 Noninformative_low11 
[137]:

len(set(corr_df.iloc[:77, ].values.reshape([-1])))

[137]:

124

[ ]:



[ ]: