【ADHD血液診断】データのダウンロードと前処理

import time
import pickle

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from scipy.stats import pearsonr
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import umap
from scipy.sparse.csgraph import connected_components

import warnings
warnings.filterwarnings('ignore')


df = pd.read_csv("src/GSE159104_DGE_raw_154samples.txt", sep="\t")
df

df.shape

df_selected_columns = df.iloc[:,1].tolist()
df_selected = df.iloc[:, 8:].T
df_selected.columns = df_selected_columns
df_selected.shape

df_selected

data = []

for i in range(df_selected.shape[0]):
    v = df_selected.iloc[i, :].tolist()
    s = np.sum(v)
    v_ = [(j/s)*(10**6) for j in v]
    data.append(v_)
    
df_rpkm = pd.DataFrame(data)
df_rpkm.index = df_selected.index
df_rpkm.columns = df_selected.columns

df_rpkm.shape

idxes = []

for i in range(df_rpkm.shape[1]):
    if not np.sum(df_rpkm.iloc[:,i]) <= 100:
        idxes.append(i)
        
df_excluded = df_rpkm.iloc[:, idxes]
df_excluded.shape

gene_set = list(set(df_excluded.columns.tolist()))

data = []

for i, g in enumerate(gene_set):
    tab = df_excluded.loc[:, g]
    if len(tab.shape) == 1:
        v = tab.tolist()
    else:
        v = df_excluded.loc[:, g].apply(lambda x: np.mean(x), axis=1).tolist()
    data.append(v)
    
    
df_gene = pd.DataFrame(data).T
df_gene.index = df_excluded.index
df_gene.columns = gene_set

df_gene.shape

non_zero_list = []

for i in range(df_gene.shape[1]):
    v = df_selected.iloc[:, i].tolist()
    s = np.std(v)
    if s != 0:
        non_zero_list.append(i)

df_std = df_gene.iloc[:, non_zero_list]

df_std.to_csv("src/df_std.csv")

df_std.shape