{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext rpy2.ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import rpy2.robjects as ro\n",
    "from rpy2.robjects import pandas2ri\n",
    "from pathlib import Path\n",
    "pandas2ri.activate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NULL\n"
     ]
    }
   ],
   "source": [
    "%%R\n",
    "#install.packages(\"BiocManager\")\n",
    "#BiocManager::install(\"DESeq2\", update=FALSE, ask=FALSE)\n",
    "#BiocManager::install(\"limma\", update=FALSE, ask=FALSE)\n",
    "#BiocManager::install(\"lumi\", update=FALSE, ask=FALSE)\n",
    "#BiocManager::install(\"qvalue\", update=FALSE, ask=FALSE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization of Study 1 dataset\n",
    "First, we will use our existing methods in R to read in our dataset from the first study and split it into healthy and controls as separate dataframes which we will hand off those dataframes to our Python code for further analysis. \n",
    "We then will select the top 25 genes based on the mean expression values in our glaucoma group. We further will adjust these in a separate list by subtracting the mean expression values of the control group to prioritize those which are not expressed in the control group."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: \n",
      "Attaching package: ‘dplyr’\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    filter, lag\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:base’:\n",
      "\n",
      "    intersect, setdiff, setequal, union\n",
      "\n",
      "\n",
      "R[write to console]: \n",
      "Attaching package: ‘MASS’\n",
      "\n",
      "\n",
      "R[write to console]: The following object is masked from ‘package:dplyr’:\n",
      "\n",
      "    select\n",
      "\n",
      "\n",
      "R[write to console]: Loading required package: Biobase\n",
      "\n",
      "R[write to console]: Loading required package: BiocGenerics\n",
      "\n",
      "R[write to console]: \n",
      "Attaching package: ‘BiocGenerics’\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:dplyr’:\n",
      "\n",
      "    combine, intersect, setdiff, union\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    IQR, mad, sd, var, xtabs\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:base’:\n",
      "\n",
      "    anyDuplicated, aperm, append, as.data.frame, basename, cbind,\n",
      "    colnames, dirname, do.call, duplicated, eval, evalq, Filter, Find,\n",
      "    get, grep, grepl, intersect, is.unsorted, lapply, Map, mapply,\n",
      "    match, mget, order, paste, pmax, pmax.int, pmin, pmin.int,\n",
      "    Position, rank, rbind, Reduce, rownames, sapply, setdiff, sort,\n",
      "    table, tapply, union, unique, unsplit, which.max, which.min\n",
      "\n",
      "\n",
      "R[write to console]: Welcome to Bioconductor\n",
      "\n",
      "    Vignettes contain introductory material; view with\n",
      "    'browseVignettes()'. To cite Bioconductor, see\n",
      "    'citation(\"Biobase\")', and for packages 'citation(\"pkgname\")'.\n",
      "\n",
      "\n",
      "R[write to console]: Setting options('download.file.method.GEOquery'='auto')\n",
      "\n",
      "R[write to console]: Setting options('GEOquery.inmemory.gpl'=FALSE)\n",
      "\n",
      "R[write to console]: No methods found in package ‘RSQLite’ for request: ‘dbListFields’ when loading ‘lumi’\n",
      "\n",
      "R[write to console]: \n",
      "Attaching package: ‘limma’\n",
      "\n",
      "\n",
      "R[write to console]: The following object is masked from ‘package:BiocGenerics’:\n",
      "\n",
      "    plotMA\n",
      "\n",
      "\n",
      "R[write to console]: Using empty ID: EMPTY-0001\n",
      "\n",
      "R[write to console]: Using empty ID: EMPTY-0001\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%R \n",
    "library(\"readxl\")\n",
    "library(\"dplyr\")\n",
    "library(\"MASS\")\n",
    "library(\"lumi\")\n",
    "library(\"limma\")\n",
    "library(\"broom\")\n",
    "library(\"qvalue\")\n",
    "dataset <- \"GLA02\"\n",
    "controls <- c(\"Anti-human IgG\", \"EBNA1\", \"Bare-bead\", \"His6ABP\")\n",
    "for (file in list.files(file.path(\"scripts\", \"backend\"))) {\n",
    "  source(file.path(\"scripts\", \"backend\", file))\n",
    "}\n",
    "\n",
    "\n",
    "### Pipeline\n",
    "import(dataset)\n",
    "stage_1(I01_Import, \"P01_Preprocessed\")\n",
    "stage_2(P01_Preprocessed, \"P02_Merged\")\n",
    "stage_3(P02_Merged, \"P03_Transformed\")\n",
    "full_dataset = P02_Merged$Set_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%R -o nameslist\n",
    "antigen_names = colnames(full_dataset[-1:-2])\n",
    "gene_names = replace_antigen_names(antigen_names)\n",
    "nameslist = list(\"antigen_names\" = antigen_names, \"gene_names\" = gene_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This next code block is just an extension of the previous one, separated for ease of reading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"There are 14 antigens in the top 50 expressed antigens that are also in the top 50 adjusted antigens\"\n"
     ]
    }
   ],
   "source": [
    "%%R -o top_50 -o adj_top_50\n",
    "healthy_dataset <- full_dataset[full_dataset$group == 0, ]\n",
    "gc_dataset <- full_dataset[full_dataset$group == 1, ]\n",
    "gc_averages <- colMeans(gc_dataset[-1:-2])\n",
    "hc_averages <- colMeans(healthy_dataset[-1:-2])\n",
    "adjusted_gc_averages <- gc_averages - hc_averages\n",
    "gc_averages <- gc_averages[order(gc_averages, decreasing = TRUE)]\n",
    "adjusted_gc_averages <- adjusted_gc_averages[order(adjusted_gc_averages, decreasing = TRUE)]\n",
    "\n",
    "top_50 <- names(gc_averages)[1:50]\n",
    "adj_top_50 <- names(adjusted_gc_averages)[1:50]\n",
    "\n",
    "#check how many of the top 25 expressed genes are in the adjusted top 25 expressed genes\n",
    "overlap = sum(top_50 %in% adj_top_50)\n",
    "print(paste(\"There are\", overlap, \"antigens in the top 50 expressed antigens that are also in the top 50 adjusted antigens\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlated Genes\n",
    "We will also check our glaucoma dataset from our first study for any genes which were strongly correlated with any of our significant seven genes. For this we will use the Pearson correlation coefficient and select those genes which have a correlation coefficient of greater than 0.75 with any of our significant genes. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%R -o correlated_antigens\n",
    "gc_corr_dataset <- P03_Transformed$Set_3\n",
    "gc_corr_dataset <- gc_corr_dataset[gc_corr_dataset$group == 1, ]\n",
    "significant_antigens <- c(\"HPRA034083\", \"HPRA000767\", \"HPRA019035\", \"HPRA006876\", \"HPRA003490\", \"HPRA022019\", \"HPRA017192\")\n",
    "\n",
    "correlation_threshold <- 0.75\n",
    "# Revised function to ensure names are preserved in the strong correlations vector\n",
    "calculate_and_filter_correlations <- function(gc_dataset, significant_antigens, corr_threshold = correlation_threshold) {\n",
    "  # Filter out non-antigen columns (keep only antigen data)\n",
    "  antigen_data <- gc_dataset[ , !(names(gc_dataset) %in% c(\"Internal.LIMS.ID\", \"group\"))]\n",
    "  \n",
    "  # Initialize a list to store strong correlation results\n",
    "  strong_correlation_results_fixed <- list()\n",
    "  \n",
    "  # Calculate the full correlation matrix once\n",
    "  full_correlations <- cor(antigen_data, use = \"complete.obs\", method = \"pearson\")\n",
    "  \n",
    "  # Iterate over each significant antigen\n",
    "  for(significant_antigen in significant_antigens) {\n",
    "    if(significant_antigen %in% names(antigen_data)) {\n",
    "      # Extract correlations for the current significant antigen\n",
    "      current_correlations <- full_correlations[, significant_antigen]\n",
    "      # Ensure names are preserved\n",
    "      names(current_correlations) <- rownames(full_correlations)\n",
    "      # Filter for strong correlations (absolute value greater than the threshold)\n",
    "      strong_correlations <- current_correlations[abs(current_correlations) > correlation_threshold]  \n",
    "      # Remove the correlation of the antigen with itself, ensuring names are used for filtering\n",
    "      strong_correlations <- strong_correlations[names(strong_correlations) != significant_antigen]\n",
    "      \n",
    "      # Store the strong correlations in the list with the antigen name as the key\n",
    "      strong_correlation_results_fixed[[significant_antigen]] <- strong_correlations\n",
    "    } else {\n",
    "      cat(paste(\"Warning: Antigen\", significant_antigen, \"not found in dataset.\\n\"))\n",
    "    }\n",
    "  }\n",
    "  \n",
    "  return(strong_correlation_results_fixed)\n",
    "}\n",
    "correlation_dataframe <- calculate_and_filter_correlations(gc_corr_dataset, significant_antigens)\n",
    "# Initialize an empty vector to store all unique antigen names\n",
    "all_strongly_correlated_antigens <- c()\n",
    "# Iterate over the list of strong correlations for each significant antigen\n",
    "for(significant_antigen in names(correlation_dataframe)) {\n",
    "  # Extract names of strongly correlated antigens for the current significant antigen\n",
    "  current_antigens <- names(correlation_dataframe[[significant_antigen]])\n",
    "  \n",
    "  # Combine with the existing list of antigen names\n",
    "  all_strongly_correlated_antigens <- c(all_strongly_correlated_antigens, current_antigens)\n",
    "}\n",
    "# Remove duplicates to get a unique list of antigen names\n",
    "all_strongly_correlated_antigens <- unique(all_strongly_correlated_antigens)\n",
    "# now replace the names with the corresponding genes and save to a list for export to python\n",
    "correlated_antigens <- all_strongly_correlated_antigens\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This part just confirms that there are not outliers in the correlation dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: \n",
      "Attaching package: ‘patchwork’\n",
      "\n",
      "\n",
      "R[write to console]: The following object is masked from ‘package:MASS’:\n",
      "\n",
      "    area\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`geom_smooth()` using formula = 'y ~ x'\n",
      "`geom_smooth()` using formula = 'y ~ x'\n",
      "`geom_smooth()` using formula = 'y ~ x'\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72480 characters]",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%R -w 1640 -h 500\n",
    "library(dplyr)\n",
    "library(patchwork)\n",
    "library(ggplot2)\n",
    "# Initialize an empty list to store pairs\n",
    "pairs_list <- list()\n",
    "\n",
    "# Loop through each list in the correlation_dataframe\n",
    "for (antigen in names(correlation_dataframe)) {\n",
    "    # Get the vector of correlations for the current antigen\n",
    "    correlations <- correlation_dataframe[[antigen]]\n",
    "    # Loop through each item in the vector\n",
    "    for (correlated_antigen in names(correlations)) {\n",
    "        # Append the pair (antigen, correlated_antigen) to the pairs_list\n",
    "        pairs_list <- append(pairs_list, list(c(antigen, correlated_antigen)))\n",
    "    }\n",
    "}\n",
    "# Now let's convert the list to a dataframe for easy handling\n",
    "pairs_df <- do.call(rbind, lapply(pairs_list, function(x) {\n",
    "    data.frame(antigen1 = x[1], antigen2 = x[2], stringsAsFactors = FALSE)\n",
    "}))\n",
    "\n",
    "# Sample 3 random rows (pairs)\n",
    "set.seed(123) # Setting a seed for reproducibility\n",
    "random_pairs <- pairs_df[sample(nrow(pairs_df), 3), ]\n",
    "# Assuming gc_corr_dataset is a data frame with columns for each antigen\n",
    "# Generate scatter plots for the 3 random pairs\n",
    "# Store the plots in a list\n",
    "plot_list <- list()\n",
    "\n",
    "for (i in 1:nrow(random_pairs)) {\n",
    "    pair <- random_pairs[i, ]\n",
    "    plot_data <- data.frame(\n",
    "        Antigen1 = gc_corr_dataset[[pair$antigen1]],\n",
    "        Antigen2 = gc_corr_dataset[[pair$antigen2]]\n",
    "    )\n",
    "\n",
    "    p <- ggplot(plot_data, aes(x = Antigen1, y = Antigen2)) +\n",
    "        geom_point(alpha = 0.6) +\n",
    "        geom_smooth(method = \"lm\", se = FALSE, color = \"blue\") +\n",
    "        labs(\n",
    "            title = paste(\"Scatter Plot with Regression Line:\", pair$antigen1, \"vs.\", pair$antigen2),\n",
    "            x = pair$antigen1,\n",
    "            y = pair$antigen2\n",
    "        ) +\n",
    "        theme_minimal()\n",
    "\n",
    "    plot_list[[i]] <- p\n",
    "}\n",
    "\n",
    "# Combine the plots side by side\n",
    "combined_plot <- plot_list[[1]] + plot_list[[2]] + plot_list[[3]] + \n",
    "                 plot_layout(ncol = 3)\n",
    "# Print the combined plot\n",
    "print(combined_plot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering and Ordering\n",
    "We then will have five lists:\n",
    "1. The seven genes which were significant in the limma testing after adjusting for multiple testing\n",
    "2. The top 50 genes based on the p-values from limma testing\n",
    "3. The top 50 genes based on the mean expression values in the glaucoma group\n",
    "4. The top 50 genes based on the mean expression values in the glaucoma group, adjusted by the mean expression values in the control group\n",
    "5. The genes which are correlated with any of the seven significant genes\n",
    "\n",
    "We will then prioritize as follows:\n",
    "1. Genes which are significant in the limma testing after adjusting for multiple testing\n",
    "2. Genes which are significant in the limma testing and in the top expression values\n",
    "3. Genes which are strongly correlated with any of the seven significant genes\n",
    "3. Genes which are in the top expression values, and adjusted for the control group, sorted by the adjusted mean expression values\n",
    "4. Genes which are in the top expression values, sorted by the mean expression values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "def read_list(file):\n",
    "    with open(file, 'r') as f:\n",
    "        return f.read().splitlines()\n",
    "#top_list = read_list(Path(\"data\", \"antigen_selection\", 'top_60.list'))\n",
    "limma_results = pd.read_excel(Path(\"data\", \"antigen_selection\", 'report.xlsx'))\n",
    "p_val_top_50 = list(limma_results['Antigen_Names'][:50])\n",
    "significant_seven = list(limma_results['Antigen_Names'][:7])\n",
    "for item in p_val_top_50.copy():\n",
    "    if item in significant_seven:\n",
    "        p_val_top_50.remove(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "expression_top = []\n",
    "top_50 = list(top_50)\n",
    "adj_top_50 = list(adj_top_50)\n",
    "#first remove the significant seven\n",
    "for item in top_50.copy():\n",
    "    if item in significant_seven:\n",
    "        top_50.remove(item)\n",
    "for item in top_50.copy():\n",
    "    if item in p_val_top_50:\n",
    "        expression_top.append(item)\n",
    "        top_50.remove(item)\n",
    "    elif item in adj_top_50:\n",
    "        expression_top.append(item)\n",
    "        top_50.remove(item)\n",
    "expression_top.extend(top_50)\n",
    "\n",
    "#we will also make sure we remove the significant seven from the correlated antigens\n",
    "for item in correlated_antigens.copy():\n",
    "    if item in significant_seven:\n",
    "        correlated_antigens.remove(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#step 1 - organize our three sources into dictionaries\n",
    "from rpy2.robjects import r, StrVector\n",
    "from rpy2.robjects.packages import importr\n",
    "def replace_antigen_names(antigens):   \n",
    "    r_vector = StrVector(antigens)\n",
    "    r.source(\"scripts/backend/data_handling.R\")\n",
    "    r_result = r.replace_antigen_names(r_vector)\n",
    "    gene_names = list(r_result)\n",
    "    return gene_names\n",
    "\n",
    "p_val_top = {\n",
    "    \"name\": \"p_val\",\n",
    "    \"antigens\": list(p_val_top_50),\n",
    "    \"genes\": replace_antigen_names(list(p_val_top_50))\n",
    "    }\n",
    "exp_top = {\n",
    "    \"name\": \"expression\",\n",
    "    \"antigens\": expression_top,\n",
    "    \"genes\": replace_antigen_names(expression_top)\n",
    "    }\n",
    "adj_top = {\n",
    "    \"name\": \"adjusted\",\n",
    "    \"antigens\": list(adj_top_50),\n",
    "    \"genes\": replace_antigen_names(list(adj_top_50))\n",
    "}\n",
    "corr_top = {\n",
    "    \"name\": \"correlation\",\n",
    "    \"antigens\": list(correlated_antigens),\n",
    "    \"genes\": replace_antigen_names(list(correlated_antigens))\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discover_highlights(dictionary_list):\n",
    "    highlight_list = []\n",
    "    for dictionary in dictionary_list:\n",
    "        for idx, gene in enumerate(dictionary[\"genes\"]):\n",
    "            #flatten the gene entry into a list\n",
    "            genes = gene.split(\",\")\n",
    "            for gene in genes:\n",
    "                highlight_entry = {\n",
    "                    \"origin\" : dictionary[\"name\"],\n",
    "                    \"antigen\" : dictionary[\"antigens\"][idx],\n",
    "                    \"gene\" : gene\n",
    "                }\n",
    "                highlight_list.append(highlight_entry)\n",
    "   \n",
    "    #get all the genes from the highlights\n",
    "    gene_list = set([highlight[\"gene\"] for highlight in highlight_list])\n",
    "    antigen_highlights = []\n",
    "    for gene in gene_list:\n",
    "        #if the gene appears in three different sources, it is a highlight\n",
    "        highlights = [highlight for highlight in highlight_list if highlight[\"gene\"] == gene]\n",
    "        sources = [highlight[\"origin\"] for highlight in highlights]\n",
    "        if len(set(sources)) >= 3:\n",
    "            #add any antigens that correspond to the gene from the highlights list\n",
    "            antigens = [highlight[\"antigen\"] for highlight in highlights if highlight[\"gene\"] == gene]\n",
    "            antigen_highlights.extend(antigens)\n",
    "    antigen_highlights = list(set(antigen_highlights))\n",
    "\n",
    "    return antigen_highlights\n",
    "\n",
    "consensus_highlights = discover_highlights([p_val_top, exp_top, adj_top, corr_top])\n",
    "consensus_highlights_genes = set(replace_antigen_names(consensus_highlights))\n",
    "\n",
    "#remove consensus highlights from other lists\n",
    "for dictionary in [p_val_top, exp_top, corr_top]:\n",
    "    for idx, analyte in enumerate(dictionary[\"antigens\"]):\n",
    "        #print(dictionary[\"name\"], idx, analyte)\n",
    "        if analyte in consensus_highlights:\n",
    "            dictionary[\"antigens\"].remove(analyte)\n",
    "            dictionary[\"genes\"].remove(dictionary[\"genes\"][idx])\n",
    "# def expand_list_items(names_list):\n",
    "#     # Split apart multiples into sublists and flatten the sublists\n",
    "#     flattened_list = [item.strip() for sublist in names_list for item in sublist.split(\",\")]\n",
    "    \n",
    "#     # Use a dict to remove duplicates while maintaining order\n",
    "#     unique_items = dict.fromkeys(flattened_list)\n",
    "    \n",
    "#     # Convert the keys of the dictionary back into a list\n",
    "#     top_list_ordered_unique = list(unique_items.keys())\n",
    "#     return top_list_ordered_unique\n",
    "\n",
    "\n",
    "# # r_vector = StrVector(expression_top)\n",
    "# # r.source(\"scripts/backend/data_handling.R\")\n",
    "# # r_result = r.replace_antigen_names(r_vector)\n",
    "\n",
    "# # python_result = list(r_result)\n",
    "# check_highlights([p_val_top, expression_top, correlation_top])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_list_items(top_list):\n",
    "    # Split apart multiples into sublists and flatten the sublists\n",
    "    flattened_list = [item.strip() for sublist in top_list for item in sublist.split(\",\")]\n",
    "    \n",
    "    # Use a dict to remove duplicates while maintaining order\n",
    "    unique_items = dict.fromkeys(flattened_list)\n",
    "    \n",
    "    # Convert the keys of the dictionary back into a list\n",
    "    top_list_ordered_unique = list(unique_items.keys())\n",
    "    return top_list_ordered_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def read_list(file):\n",
    "#     with open(file, 'r') as f:\n",
    "#         return f.read().splitlines()\n",
    "# top_list = read_list(Path(\"data\", \"antigen_selection\", 'top_60.list'))\n",
    "# significant_seven = [\"DGCR2\", \"LOX\", \"FUT2\", \"LGSN\", \"ANXA10\", \"TMEM9B\", \"CDH5\"]\n",
    "# #remove the significant seven from the top_list\n",
    "# top_list = [x for x in top_list if x not in significant_seven]\n",
    "# p_val_top_50 = top_list[:50]\n",
    "# #some of the items in this list are actually multiple genes, so we need to fix that\n",
    "\n",
    "\n",
    "# p_val_top_50 = expand_list_items(p_val_top_50)\n",
    "# expression_val_top_50 = expand_list_items(list(top_50).copy())\n",
    "# adjusted_top_50 = expand_list_items(list(adj_top_50).copy())\n",
    "# correlated_gene_list = expand_list_items(correlated_genes.copy())\n",
    "# combined_list =[]\n",
    "# #remove the significant seven from the expression_val_top_50\n",
    "# expression_val_top_50 = [x for x in expression_val_top_50 if x not in significant_seven]\n",
    "# #remove the significant seven from the adjusted_top_50\n",
    "# adjusted_top_50 = [x for x in adjusted_top_50 if x not in significant_seven]\n",
    "# #check if any items appear in exp_top_50 and p_top_50 and adjusted_top_50\n",
    "# highlight_list = list((set(expression_val_top_50) & set(p_val_top_50) & set(adjusted_top_50)))\n",
    "# # highlight_list = []\n",
    "# for item in highlight_list:\n",
    "#     expression_val_top_50.remove(item)\n",
    "#     p_val_top_50.remove(item)\n",
    "#     adjusted_top_50.remove(item)\n",
    "\n",
    "# #In order to prioritize the items which appear in two metrics over the items which appear in one metric, we will do some list sorting\n",
    "# #First we will sort the items which appear in two metrics\n",
    "# p_e, a_e = [], []\n",
    "# for item in expression_val_top_50:\n",
    "#     if item in p_val_top_50:\n",
    "#         p_e.append(item)\n",
    "#         p_val_top_50.remove(item)\n",
    "#         expression_val_top_50.remove(item)\n",
    "#     elif item in adjusted_top_50:\n",
    "#         a_e.append(item)\n",
    "#         adjusted_top_50.remove(item)\n",
    "#         expression_val_top_50.remove(item)\n",
    "# #Then we will compile our expression list as a \"zipper\" of the two lists, followed by anything remaining in the expression_val_top_50 list (which only appears in one metric)\n",
    "# #Note that appearing in p_val_top_50 is prioritized over appearing in adjusted_top_50, but only by one index position in our final list\n",
    "# expression_list = []\n",
    "# for i in range(max(len(p_e), len(a_e))):\n",
    "#     if i < len(p_e):\n",
    "#         expression_list.append(p_e[i])\n",
    "#     if i < len(a_e):\n",
    "#         expression_list.append(a_e[i])\n",
    "\n",
    "# expression_list += expression_val_top_50\n",
    "\n",
    "# #Finally, since our correlation genes are lower priority than the highlight list, but higher priority than the expression list, we will remove them acccording to the same logic\n",
    "# for item in highlight_list:\n",
    "#     if item in correlated_gene_list:\n",
    "#         correlated_gene_list.remove(item)\n",
    "# for item in correlated_gene_list:\n",
    "#     if item in p_val_top_50:\n",
    "#         p_val_top_50.remove(item)\n",
    "#     elif item in expression_list:\n",
    "#         expression_list.remove(item)\n",
    "# print(set(expression_list) & set(p_val_top_50))\n",
    "# ##For some reason SGK494 wants to overlap in our sets so we are just gonna remove it from the expression list\n",
    "# expression_list.remove(\"SGK494\")\n",
    "# print(set(expression_list) & set(p_val_top_50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Result \n",
    "These two we definitely want because every metric we use here identifies these in the top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['HPRA022890',\n",
       " 'HPRA024016',\n",
       " 'HPRA022891',\n",
       " 'HPRA042866',\n",
       " 'HPRA045993',\n",
       " 'HPRA020522',\n",
       " 'HPRA021736',\n",
       " 'HPRA046255',\n",
       " 'HPRA019334',\n",
       " 'HPRA020521',\n",
       " 'HPRA013060',\n",
       " 'HPRA022506',\n",
       " 'HPRA019333',\n",
       " 'HPRA013851',\n",
       " 'HPRA013573',\n",
       " 'HPRA042751',\n",
       " 'HPRA017982',\n",
       " 'HPRA021737',\n",
       " 'HPRA045372',\n",
       " 'HPRA029223',\n",
       " 'HPRA001197',\n",
       " 'HPRA004387',\n",
       " 'HPRA002363',\n",
       " 'HPRA026116',\n",
       " 'HPRA011757',\n",
       " 'HPRA037260']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "consensus_highlights "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary of Lists\n",
    "Now we have three lists containing:\n",
    "1. The genes which were highlighted by all metrics\n",
    "2. The genes which were highly correlated with the significant genes\n",
    "3. The genes which were highlighted by the top expression values\n",
    "4. The genes which were highlighted by the top p values\n",
    "\n",
    "All of these list are ordered by their respective values, so we can easily drop from the end of the list based on the number of genes we want to keep. \n",
    "\n",
    "We actually have two more things to add to this set of lists.\n",
    "\n",
    "5. Genes which are interesting based on other studies, but not necessarily in our lists\n",
    "6. Genes which are highlighted by network analysis using our graphnet package\n",
    "\n",
    "After that, we can adjust the number of genes we keep from numbers 2,3, and 5 here. Numbers 1 and 4 will be kept in their entirety."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#These genes are highlighted by the corresponding journals for their connection with xfg\n",
    "other_studies_dictionary = {\n",
    "    \"CACNA1A\": \"https://journals.lww.com/glaucomajournal/fulltext/2018/07001/mechanisms_of_glaucoma_in_exfoliation_syndrome.17.aspx\",\n",
    "    \"AGPAT1\": \"https://pubmed.ncbi.nlm.nih.gov/31687947/\",\n",
    "    \"CYP39A1\": \"https://pubmed.ncbi.nlm.nih.gov/34763023/\",\n",
    "    \"POMP\": \"https://www.nature.com/articles/ng.3875\",\n",
    "    \"TMEM136\": \"https://www.nature.com/articles/ng.3875\",\n",
    "    \"RBMS3\": \"https://www.nature.com/articles/ng.3875\",\n",
    "    \"SEMA6A\": \"https://www.nature.com/articles/ng.3875\",\n",
    "    \"LOXL1\": \"addurllater\",\n",
    "}\n",
    "#We will double check that these genes do not appear in any of the lists, if they do, they can be removed from the dictionary\n",
    "for gene in other_studies_dictionary.keys():\n",
    "    if gene in expand_list_items(exp_top['genes']) or gene in expand_list_items(p_val_top['genes']) or gene in expand_list_items(corr_top['genes']):\n",
    "        print(f\"{gene} appears in one of the lists\")\n",
    "        other_studies_dictionary.pop(gene)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network Interaction\n",
    "\n",
    "### Determining counts\n",
    "Before we start the interaction analysis, we need to know how many we will keep from each list.\n",
    "We know that we will keep all of the significant seven, and all of the other studies genes.\n",
    "We also will definitely keep the genes that are in our consensus highlights.\n",
    "We have 92 total to choose, so lets work from there.\n",
    "We will choose a ratio of 40% from the network interaction list, and 60% from the other lists, divided evenly among metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will choose 21 genes from the network interaction list, 10 genes from the expression list, 10 genes from the p_value list, and 10 genes from the correlation list\n"
     ]
    }
   ],
   "source": [
    "ratio = [0.40, 0.20, 0.2, .2]\n",
    "reserved_count = len(significant_seven) + len(other_studies_dictionary) + len(consensus_highlights) \n",
    "remaining_count = 92 - reserved_count\n",
    "\n",
    "#now we can multiply each ratio by the remaining count to get the number of genes we will choose from each list\n",
    "network_interaction_count = int(remaining_count * ratio[0])\n",
    "expression_count = int(remaining_count * ratio[1])\n",
    "p_value_count = int(remaining_count * ratio[2])\n",
    "correlation_count = int(remaining_count * ratio[3])\n",
    "#if the sum of these counts is less than the remaining count, we will add the difference to the network interaction count\n",
    "difference = remaining_count - (network_interaction_count + expression_count + p_value_count + correlation_count)\n",
    "#distribute any difference evenly across the lists\n",
    "for i in range(difference):\n",
    "    if i % 4 == 0:\n",
    "        network_interaction_count += 1\n",
    "    elif i % 4 == 1:\n",
    "        expression_count += 1\n",
    "    elif i % 4 == 2:\n",
    "        p_value_count += 1\n",
    "    else:\n",
    "        correlation_count += 1\n",
    "\n",
    "print(f\"We will choose {network_interaction_count} genes from the network interaction list, {expression_count} genes from the expression list, {p_value_count} genes from the p_value list, and {correlation_count} genes from the correlation list\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "import collections\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from modules import Graphnet\n",
    "\n",
    "keychain = Path(\"keys\", \".keychain\")\n",
    "keys = json.load(open(keychain))\n",
    "graph = Graphnet(\n",
    "    uri=keys[\"graphnet\"][\"uri\"],\n",
    "    username=keys[\"graphnet\"][\"username\"],\n",
    "    password=keys[\"graphnet\"][\"password\"],\n",
    ")\n",
    "#Now we will set our limits for the number of genes we will choose from each list\n",
    "expression_aggregate = expand_list_items(exp_top['genes'])\n",
    "p_value_aggregate = expand_list_items(p_val_top['genes'])\n",
    "correlation_aggregate = expand_list_items(corr_top['genes'])\n",
    "s7_aggregate = set(replace_antigen_names(significant_seven))\n",
    "highlight_aggregate = set(replace_antigen_names(consensus_highlights))\n",
    "\n",
    "# #Then we will join all of the items we are keeping, except the other studies genes, into one list for use by our network interaction model\n",
    "aggregate_top_list = []\n",
    "aggregate_top_list.extend(s7_aggregate)\n",
    "aggregate_top_list.extend(highlight_aggregate)\n",
    "iteration = max(len(expression_aggregate), len(p_value_aggregate), len(correlation_aggregate))\n",
    "for i in range(iteration):\n",
    "    if i < len(expression_aggregate):\n",
    "        aggregate_top_list.append(expression_aggregate[i])\n",
    "    if i < len(p_value_aggregate):\n",
    "        aggregate_top_list.append(p_value_aggregate[i])\n",
    "    if i < len(correlation_aggregate):\n",
    "        aggregate_top_list.append(correlation_aggregate[i])\n",
    "aggregate_top_list = expand_list_items(aggregate_top_list)[0:50]\n",
    "# aggregate_top_list.extend(significant_seven)\n",
    "# aggregate_top_list.extend(highlight_list)\n",
    "# aggregate_top_list.extend(correlated_gene_list)\n",
    "# current_length = len(aggregate_top_list)\n",
    "# target_length = 92 - (len(other_studies_dictionary) + network_interaction_count)\n",
    "# #zipper in expression and p_value lists up to 40 items total. This prevents the network interaction list from being too long based on the lower content of our lists\n",
    "# #subtracted 4 additionally because the numbers are not working out perfectly\n",
    "# for i in range(0, int((target_length-current_length)/2)-4):\n",
    "#     #print(i)\n",
    "#     if i < len(expression_list):\n",
    "#         aggregate_top_list.append(expression_list[i])\n",
    "#     if i < len(p_value_list):\n",
    "#         aggregate_top_list.append(p_value_list[i])\n",
    "\n",
    "\n",
    "#Just to double check, we should expect a length of 92 - (the length of the other studies dictionary+the length of the planned network interaction list)\n",
    "# print(len(aggregate_top_list) == 92 - (len(other_studies_dictionary) + network_interaction_count) )   #True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Interaction Dictionaries for Nodes\n",
    "\n",
    "In this analysis, we aim to explore the interactions between a set of proteins. For each protein (or \"node\") in our list of interest (`aggregate_top_list`), we construct an interaction dictionary. This dictionary maps out the shortest paths between the start node and each node of interest, highlighting the key proteins (or \"intermediates\") involved. Particularly, we focus on identifying proteins that appear across multiple paths, referred to as \"prime_duplicates.\" These prime duplicates may suggest significant roles in protein interaction networks.\n",
    "\n",
    "### Function Overview\n",
    "\n",
    "The core of our analysis relies on the `build_interaction_dictionary` function. This function generates a dictionary of proteins along the shortest paths between a start node and nodes of interest, using our graph network based on Neo4j.\n",
    "\n",
    "\n",
    "This function processes a given start node against a list of nodes of interest, identifying all relevant proteins along the shortest paths between those nodes and pinpointing any significant overlaps as prime duplicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_interaction_dictionary(start_node:str, nodes_of_interest:list):\n",
    "\n",
    "    \"\"\"\n",
    "    This function takes a start node and a list of nodes of interest and returns a dictionary of the proteins appearing along the shortest paths between the start node and each node of interest.\n",
    "    Additionally, identifies any proteins that appear in multiple paths as \"prime_duplicates\"\n",
    "\n",
    "    Args:\n",
    "    start_node: The protein_ID of the start node\n",
    "    nodes_of_interest: A list of protein_IDs of the nodes of interest\n",
    "\n",
    "    Returns:\n",
    "    node_dict: A dictionary containing the start node, the intermediate interactions, and any prime_duplicates\n",
    "\n",
    "    \"\"\"\n",
    "    node_paths_list = []\n",
    "    for end_node in nodes_of_interest:\n",
    "        if end_node == start_node:\n",
    "            continue\n",
    "        #print(f'Finding path from {start_node} to {end_node}')\n",
    "        path = graph.find_shortest_path(start_node=start_node, end_node=end_node, min_strength=50)\n",
    "        actual_path = path[0][\"path\"]\n",
    "        path_nodes = [node[\"protein_ID\"] for node in actual_path.nodes]\n",
    "        #remove the start and end nodes from the list\n",
    "        path_nodes.remove(start_node)\n",
    "        path_nodes.remove(end_node)\n",
    "        #if the path is not empty, print the path\n",
    "        if path_nodes != []:\n",
    "            #print(f'Path from {start_node} to {end_node} is {path_nodes}')\n",
    "            node_paths_list.append(path_nodes)\n",
    "        #else:\n",
    "            #print(f'The nodes {start_node} and {end_node} directly interact')\n",
    "    #flatten the list\n",
    "    node_paths_list = [item for sublist in node_paths_list for item in sublist]\n",
    "    #print any duplicates in the list\n",
    "    prime_duplicates = [item for item, count in collections.Counter(node_paths_list).items() if count > 1]\n",
    "    node_dict = {\n",
    "        \"node\": start_node,\n",
    "        \"intermediate_interactions\": node_paths_list,\n",
    "        \"prime_duplicates\": prime_duplicates\n",
    "    }\n",
    "    return node_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation\n",
    "\n",
    "First, we need to check that all the items in our `aggregate_top_list` are available in our database and remove any that are unsupported. We will print these out so that we know what has been excluded. We then iterate through each node in `aggregate_top_list`, applying `build_interaction_dictionary` to uncover the network of interactions. The following code snippet demonstrates this process, with a status update displaying the progress of this computation.\n",
    "\n",
    "Finally we can a quick description of the dataframe just to see that we have results here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGK494 is not in the database\n",
      "AC005726.1 is not in the database\n",
      "PALM2-AKAP2 is not in the database\n",
      "PALM2 is not in the database\n"
     ]
    }
   ],
   "source": [
    "unfound_items = []\n",
    "for idx, item in enumerate(aggregate_top_list):\n",
    "    #print(idx, item)\n",
    "    checklist = (graph.execute_query(f\"MATCH (p:protein) WHERE p.protein_ID = '{item}' RETURN p.protein_ID, p.annotation\"))\n",
    "    if checklist:\n",
    "        pass#print(f\"{item} is in the database\")\n",
    "    else:\n",
    "        print(f\"{item} is not in the database\")\n",
    "        #print(f'{idx}, {item} not found')\n",
    "        unfound_items.append((item, idx))\n",
    "#remove the unfound items from the list based on the index\n",
    "for item in unfound_items[::-1]:\n",
    "    #drop the item from the list\n",
    "    aggregate_top_list.pop(item[1])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       node                          intermediate_interactions  \\1\n",
      "count    46                                                 46   \n",
      "unique   46                                                 46   \n",
      "top     LOX  [MFSD2A, HSPB6, MRI1, VEGFB, ACTB, HYOU1, ARPC...   \n",
      "freq      1                                                  1   \n",
      "\n",
      "           prime_duplicates  \n",
      "count                    46  \n",
      "unique                   45  \n",
      "top     [HSPB6, CFTR, ARF5]  \n",
      "freq                      2  \n"
     ]
    }
   ],
   "source": [
    "interaction_df = pd.DataFrame()\n",
    "total_nodes = len(aggregate_top_list)\n",
    "for index, start_node in enumerate(aggregate_top_list):\n",
    "    percentage_complete = (index + 1) / total_nodes * 100  # Calculate completion percentage\n",
    "    print(f\"Percentage complete: {percentage_complete:.2f}% | Currently processing: {start_node}\", end='\\r')\n",
    "    node_dict = build_interaction_dictionary(start_node, aggregate_top_list)\n",
    "    # Note: It seems there might be a typo in your code `_append` should be `append`\n",
    "    interaction_df = interaction_df._append(node_dict, ignore_index=True)\n",
    "    time.sleep(0.1) #added to prevent lockup of the database due to too many requests\n",
    "print(interaction_df.describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking for overlaps in the interaction dictionaries  \n",
    "\n",
    "Next we will iterate through our interaction dictionaries and look for overlaps in the proteins that are found in the shortest paths between the start node and the nodes of interest. We will then identify the proteins that are found in the most interaction dictionaries and consider these as potential antigens for the xMAP assay.  \n",
    "\n",
    "We chose to first select the significant proteins from our previous study which had adjusted p-valuse less than 0.05 and highlight the overlapping proteins in these interaction dictionaries over the rest of the list.\n",
    "\n",
    "We also note whether any of the proteins in the prime duplicates appear in multiple entries in the top list. This may suggest that these proteins are especially significant and should be considered for the xMAP assay.\n",
    "\n",
    "#### Clarification on Primes\n",
    "As a definition, herein we refer to \"prime duplicates\" as proteins which appear in the shortest path from a given starting node to other nodes within a given subset. These prime duplicates are of particular interest, as they may suggest significant roles in protein interaction networks.\n",
    "\n",
    "\"significant primes\" is meant to reflect an added value, such as if these prime duplicates are derived from a starting node from our \"significant seven\" list, or if they are \"prime duplicates\" that appear based on multiple starting nodes. Note: this is not a formal term, but rather a way to distinguish between prime duplicates that are of particular interest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Duplicate prime duplicates: ['HSPB6', 'CLTC', 'ACTB']\n",
      "Prime duplicates for the significant seven: ['SOD3', 'CLTC', 'S100A2', 'TMEM59L', 'SOX5', 'HYOU1', 'ENDOV', 'HSPB6', 'PDK4', 'ACTB', 'SERPINH1', 'VCP', 'BBS10', 'CFTR', 'AUP1', 'PHGDH', 'RNF150', 'MAPK8IP3']\n",
      "Total number of significant primes: 18\n"
     ]
    }
   ],
   "source": [
    "#check the prime_duplicates for the significant seven\n",
    "significant_primes = []\n",
    "for node in list(s7_aggregate):\n",
    "    #print(f\"Prime duplicates for {node}: {interaction_df[interaction_df['node'] == node]['prime_duplicates'].values[0]}\")\n",
    "    significant_primes.append(interaction_df[interaction_df['node'] == node]['prime_duplicates'].values[0])\n",
    "#flatten the list\n",
    "significant_primes = [item for sublist in significant_primes for item in sublist]\n",
    "#print any duplicates in the list\n",
    "multi_sig_primes = [item for item, count in collections.Counter(significant_primes).items() if count > 1]\n",
    "print(\"\")\n",
    "print(f\"Duplicate prime duplicates: {multi_sig_primes}\")\n",
    "#remove duplicates\n",
    "significant_primes = list(set(significant_primes))\n",
    "#double check that none of the significant primes are in the aggregate_top_list\n",
    "for gene in significant_primes:\n",
    "    if gene in aggregate_top_list:\n",
    "        print(f\"{gene} is in the aggregate_top_list\")\n",
    "        significant_primes.remove(gene)\n",
    "print(f\"Prime duplicates for the significant seven: {significant_primes}\")\n",
    "#add the multi_sig_primes to the significant_primes\n",
    "significant_primes.extend(multi_sig_primes)\n",
    "#remove any duplicates\n",
    "significant_primes = list(set(significant_primes))\n",
    "print(f\"Total number of significant primes: {len(significant_primes)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expanding to the full list\n",
    "\n",
    "We repeat the process again for the rest of the proteins in the top list with the exception that we also will count the number of times that the proteins in the prime duplicates appear in the interaction dictionaries. This will allow us to select the most likely candidates for the xMAP assay.\n",
    "Here we exclude any proteins already in the top list from the previous step, and select an \"n\" value for how many additional proteins we would like to include, which will be the remainder of our network interaction count, minus the number of prime duplicates already discovered.\n",
    "The printed value shows the protein names as well as the number of starting nodes they appear as prime duplicates for. This will allow us to select the most likely candidates for the xMAP assay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The top 3 prime duplicates are:\n",
      "ARF5 : 9\n",
      "CTNNB1 : 7\n",
      "GRIA3 : 4\n"
     ]
    }
   ],
   "source": [
    "# Number of top items to print\n",
    "n = network_interaction_count- len(significant_primes)\n",
    "\n",
    "secondary_significant_primes = [item for item in interaction_df['prime_duplicates']]\n",
    "secondary_significant_primes = [item for sublist in secondary_significant_primes for item in sublist]\n",
    "secondary_multi_sig_primes = [(item,count) for item, count in collections.Counter(secondary_significant_primes).items() if count > 1]\n",
    "new_additions = [item for item in secondary_multi_sig_primes if item[0] not in significant_primes]\n",
    "new_additions.sort(key=lambda x: x[1], reverse=True)\n",
    "#double check that none of the new additions are in the aggregate_top_list or the significant_primes\n",
    "for prime, count in new_additions:\n",
    "    if prime in aggregate_top_list or prime in significant_primes:\n",
    "        print(f\"{prime} is in the aggregate_top_list or the significant_primes\")\n",
    "        new_additions.remove((prime, count))\n",
    "\n",
    "print(f\"The top {n} prime duplicates are:\")\n",
    "for prime, count in new_additions[:n]:\n",
    "    print(f\"{prime} : {count}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Answer\n",
    "\n",
    "Finally, we will combine our lists:\n",
    "1. The significant proteins from the previous study\n",
    "2. The proteins that appeared in the most interaction dictionaries based on the significant seven\n",
    "3. The proteins that appeared in the most interaction dictionaries based on the rest of the top list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AGPAT1 is in other_studies at index 1\n",
      "DCUN1D4 is in new_additions at index 4\n",
      "DCUN1D4 is in new_additions at index 4\n",
      "PHGDH is in significant_primes at index 8\n",
      "PHGDH is in significant_primes at index 8\n",
      "TMEM59L is in significant_primes at index 3\n",
      "ENDOV is in significant_primes at index 17\n",
      "POMP is in other_studies at index 3\n",
      "POMP is in other_studies at index 3\n",
      "VCP is in significant_primes at index 11\n",
      "VCP is in significant_primes at index 11\n",
      "AUP1 is in significant_primes at index 14\n",
      "AUP1 is in significant_primes at index 14\n",
      "GAS7 is in new_additions at index 6\n",
      "GAS7 is in new_additions at index 6\n",
      "RBMS3 is in other_studies at index 5\n",
      "ACTB is in exp_top at index 49\n",
      "ACTB is in significant_primes at index 15\n",
      "ACTB is in exp_top at index 49\n",
      "ACTB is in significant_primes at index 15\n",
      "CYP39A1 is in other_studies at index 2\n",
      "PDK4 is in significant_primes at index 7\n",
      "CACNA1A is in other_studies at index 0\n",
      "CACNA1A is in other_studies at index 0\n",
      "HSPB6 is in significant_primes at index 6\n",
      "HSPB6 is in significant_primes at index 6\n",
      "S100A2 is in significant_primes at index 2\n",
      "CFTR is in significant_primes at index 13\n",
      "CFTR is in significant_primes at index 13\n",
      "HYOU1 is in significant_primes at index 5\n",
      "SEMA6A is in other_studies at index 6\n",
      "CLTC is in significant_primes at index 1\n",
      "KIF1A is in new_additions at index 3\n",
      "KIF1A is in new_additions at index 3\n",
      "SERPINH1 is in significant_primes at index 9\n",
      "CRYAB is in new_additions at index 5\n",
      "LOXL1 is in other_studies at index 7\n",
      "LOXL1 is in other_studies at index 7\n",
      "SOD3 is in significant_primes at index 0\n",
      "CTNNB1 is in new_additions at index 1\n",
      "CTNNB1 is in new_additions at index 1\n",
      "MAPK8IP3 is in significant_primes at index 10\n",
      "MAPK8IP3 is in significant_primes at index 10\n",
      "SOX5 is in significant_primes at index 4\n",
      "BBS10 is in significant_primes at index 12\n",
      "GRIA3 is in new_additions at index 2\n",
      "GRIA3 is in new_additions at index 2\n",
      "RNF150 is in significant_primes at index 16\n"
     ]
    }
   ],
   "source": [
    "new_additions_genes = [n[0] for n in new_additions]\n",
    "exp_top_genes = expand_list_items(exp_top['genes'])\n",
    "p_val_top_genes = expand_list_items(p_val_top['genes'])\n",
    "corr_top_genes = expand_list_items(corr_top['genes'])\n",
    "other_studies_genes = list(other_studies_dictionary.keys())\n",
    "significant_primes = list(significant_primes)\n",
    "final_list_genes = final_list_genes\n",
    "\n",
    "##I want to build a quick tool that will allow me to check which lists a given gene is in and give me the indexes in those lists\n",
    "def gene_checker(genecol):\n",
    "    for gene in genecol:\n",
    "        for listname, genes_list in {\n",
    "            \"new_additions\" : new_additions_genes, \n",
    "            \"exp_top\" : exp_top_genes, \n",
    "            \"p_val_top\": p_val_top_genes, \n",
    "            \"corr_top\": corr_top_genes, \n",
    "            # \"final_list\" : final_list_genes,\n",
    "            \"other_studies\" : other_studies_genes,\n",
    "            \"significant_primes\" : significant_primes\n",
    "            }.items():\n",
    "            if gene in genes_list:\n",
    "                #find the index of the gene in the list\n",
    "                index = genes_list.index(gene)\n",
    "                print(f\"{gene} is in {listname} at index {index}\")\n",
    "\n",
    "genecol = [\"AGPAT1\", \"DCUN1D4\", \"DCUN1D4\", \"PHGDH\", \"PHGDH\", \"TMEM59L\", \"ENDOV\", \"POMP\", \"POMP\", \"VCP\", \"VCP\", \"AUP1\", \"AUP1\", \"GAS7\", \"GAS7\", \"RBMS3\", \"ACTB\", \"ACTB\", \"CYP39A1\", \"PDK4\", \"CACNA1A\", \"CACNA1A\", \"HSPB6\", \"HSPB6\", \"S100A2\", \"CFTR\", \"CFTR\", \"HYOU1\", \"SEMA6A\", \"CLTC\", \"KIF1A\", \"KIF1A\", \"SERPINH1\", \"CRYAB\", \"LOXL1\", \"LOXL1\", \"SOD3\", \"CTNNB1\", \"CTNNB1\", \"MAPK8IP3\", \"MAPK8IP3\", \"SOX5\", \"BBS10\", \"GRIA3\", \"GRIA3\", \"RNF150\"]\n",
    "gene_checker(genecol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The final list contains 81 unique antigens and genes\n",
      "We will add 11 items evenly from the different metrics\n",
      "Added 3 items from p_val_top\n",
      "Added 4 items from exp_top\n",
      "Added 0 items from corr_top\n",
      "Added 4 items from new_additions\n",
      "The final list now contains 92 unique antigens and genes\n"
     ]
    }
   ],
   "source": [
    "# Adding it all together now\n",
    "final_list_antigens = []\n",
    "final_list_genes = []\n",
    "#First we add our reserved items\n",
    "final_list_antigens.extend(significant_seven)\n",
    "final_list_antigens.extend(consensus_highlights)\n",
    "final_list_genes.extend(other_studies_dictionary.keys())\n",
    "#Then we add from each of our metrics the intended amount of items\n",
    "final_list_antigens.extend(p_val_top['antigens'][0:p_value_count])\n",
    "final_list_antigens.extend(exp_top['antigens'][0:expression_count])\n",
    "final_list_antigens.extend(corr_top['antigens'][0:correlation_count])\n",
    "final_list_genes.extend(significant_primes)\n",
    "final_list_genes.extend([item[0] for item in new_additions[:n]])\n",
    "\n",
    "#Now we check the length of the final list, after removing any duplicates\n",
    "final_list = list(set(final_list_antigens))+list(set(final_list_genes))\n",
    "print(f\"The final list contains {len(final_list)} unique antigens and genes\")\n",
    "addition_length = 92 - len(final_list)\n",
    "print(f\"We will add {addition_length} items evenly from the different metrics\")\n",
    "# Initialize counters\n",
    "p_count = 0\n",
    "e_count = 0\n",
    "c_count = 0\n",
    "n_count = 0\n",
    "i=1\n",
    "while len(set(final_list_antigens)) + len(set(final_list_genes)) < 92:\n",
    "    final_list_antigens = list(set(final_list_antigens))\n",
    "    final_list_genes = list(set(final_list_genes))\n",
    "    position_add = i // 4\n",
    "    if i % 4 == 0:\n",
    "        if len(p_val_top['antigens']) > p_value_count + position_add:\n",
    "            antigen = p_val_top['antigens'][p_value_count + position_add]\n",
    "            if antigen not in final_list_antigens:\n",
    "                final_list_antigens.append(antigen)\n",
    "                p_count += 1  # Increment counter\n",
    "    elif i % 4 == 1:\n",
    "        if len(exp_top['antigens']) > expression_count + position_add:\n",
    "            antigen = exp_top['antigens'][expression_count + position_add]\n",
    "            if antigen not in final_list_antigens:\n",
    "                final_list_antigens.append(antigen)\n",
    "                e_count += 1  # Increment counter\n",
    "    elif i % 4 == 2:\n",
    "        if len(corr_top['antigens']) > correlation_count + position_add:\n",
    "            antigen = corr_top['antigens'][correlation_count + position_add]\n",
    "            if antigen not in final_list_antigens:\n",
    "                final_list_antigens.append(antigen)\n",
    "                c_count += 1  # Increment counter\n",
    "    else:\n",
    "        if len(new_additions) > n + position_add:\n",
    "            gene = new_additions[n + position_add][0]\n",
    "            if gene not in final_list_genes:\n",
    "                final_list_genes.append(gene)\n",
    "                n_count += 1  # Increment counter\n",
    "    i += 1\n",
    "\n",
    "# Print the counts\n",
    "print(f\"Added {p_count} items from p_val_top\")\n",
    "print(f\"Added {e_count} items from exp_top\")\n",
    "print(f\"Added {c_count} items from corr_top\")\n",
    "print(f\"Added {n_count} items from new_additions\")\n",
    "\n",
    "final_list = list(set(final_list_antigens))+list(set(final_list_genes))\n",
    "print(f\"The final list now contains {len(final_list)} unique antigens and genes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_list = significant_primes + highlight_list + list(other_studies_dictionary.keys()) + significant_seven + new_additions + correlated_gene_list + expression_list + p_value_list \n",
    "# #One last check that we don't have any duplicates\n",
    "# print(len(set(final_list)) == len(final_list)) #True\n",
    "\n",
    "items_per_line = 10  # Set the number of items per line\n",
    "#sort final_lists alphabetically\n",
    "final_list_antigens.sort()\n",
    "final_list_antigens_gene_names = replace_antigen_names(final_list_antigens)\n",
    "final_list_genes.sort()\n",
    "#create a table of the final list antigens with their corresponding genes\n",
    "final_list_table = pd.DataFrame(list(zip(final_list_antigens, final_list_antigens_gene_names)), columns=['Antigen', 'Corresponding Gene'])\n",
    "with open('selection.txt', 'w') as f:\n",
    "    print(final_list_table.to_string(index=False), file=f)\n",
    "    print(\"\\n\\n\", file=f)\n",
    "    print(\"Selected additional genes to target (need antigens for):\", file=f)\n",
    "    for i in range(0, len(final_list_genes), items_per_line):\n",
    "        print(', '.join(final_list_genes[i:i + items_per_line]), file=f)\n",
    "# for i in range(0, len(final_list), items_per_line):\n",
    "#     print(', '.join(final_list[i:i + items_per_line]))\n",
    "# print(f\"Total selected: {len(final_list)}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overlap between final_list_genes and final_list_antigens_gene_names: set()\n"
     ]
    }
   ],
   "source": [
    "#check if any items in the final_list_antigen_gene_names are also in the final_list_genes\n",
    "final_list_genes = set(final_list_genes)\n",
    "final_list_antigens_gene_names = set(final_list_antigens_gene_names)\n",
    "overlap = final_list_genes.intersection(final_list_antigens_gene_names)\n",
    "print(f\"Overlap between final_list_genes and final_list_antigens_gene_names: {overlap}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ADK',\n",
       " 'ANKRD6',\n",
       " 'IQSEC1',\n",
       " 'PTGES3L-AARSD1',\n",
       " 'PTGES3L',\n",
       " 'BTBD6',\n",
       " 'SELE',\n",
       " 'CAPZA2',\n",
       " 'CAPZA1',\n",
       " 'VAV2',\n",
       " 'CCDC83',\n",
       " 'IL6R',\n",
       " 'AARSD1',\n",
       " 'TBC1D9B']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# #join our three dictionaries\n",
    "# final_dictionary = {**expression_dict, **p_val_top_50_names_dict, **correlation_dict}\n",
    "# final_antigens = []\n",
    "# for gene in final_list:\n",
    "#     matches = [antigen for antigen, gene_name in final_dictionary.items() if gene_name == gene]\n",
    "#     print(matches)\n",
    "expand_list_items(consensus_highlights_genes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xmap_study_2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
