LLM Integration for Plots

import os
import subprocess
import sys
from pathlib import Path
from typing import List, Dict, Optional
import pandas as pd
import numpy as np
from latch.ldata.path import LPath

# Install required packages
subprocess.check_call([sys.executable, "-m", "pip", "install", "scanpy"])
import scanpy as sc

# Check if combined data exists locally
if os.path.exists("combined_data_patients_all.h5ad"):
    print("Found existing combined data file. Loading...")
    combined = sc.read_h5ad("combined_data_patients_all.h5ad")
    print(f"Loaded combined AnnData object: {combined.shape[0]} cells, {combined.shape[1]} genes")
else:
    # Function to check if a directory contains files ending with required suffixes
    def is_valid_sc_directory(dir_path: LPath) -> bool:
        """Check if directory contains files ending with required suffixes"""
        files = list(dir_path.iterdir())
        required_suffixes = ['matrix.mtx', 'barcodes.tsv', 'features.tsv']
        return all(any(f.path.endswith(suffix) or f.path.endswith(f"{suffix}.gz") for f in files) for suffix in required_suffixes)

    # Function to process a single sample
    def process_sample(sample_dir: LPath, sample_id: str) -> Optional[sc.AnnData]:
        """Process a single sample directory and return AnnData object"""
        try:
            # Create a temporary directory for this sample
            temp_dir = Path(f"/tmp/{sample_id}")
            os.makedirs(temp_dir, exist_ok=True)

            # Download and rename files
            for file in sample_dir.iterdir():
                if 'matrix.mtx' in file.path or 'barcodes.tsv' in file.path or 'features.tsv' in file.path:
                    # Download with original name
                    local_path = temp_dir / file.name()
                    file.download(local_path)
                    
                    # Rename to standard 10x names
                    new_name = None
                    if 'matrix.mtx' in file.path:
                        new_name = 'matrix.mtx.gz'
                    elif 'barcodes.tsv' in file.path:
                        new_name = 'barcodes.tsv.gz'
                    elif 'features.tsv' in file.path:
                        new_name = 'features.tsv.gz'
                    
                    if new_name and local_path.exists():
                        os.rename(local_path, temp_dir / new_name)

            # Check if all required files exist
            required_files = ['matrix.mtx.gz', 'barcodes.tsv.gz', 'features.tsv.gz']
            if not all((temp_dir / f).exists() for f in required_files):
                print(f"Missing required files in {sample_id}")
                return None

            # Read the sample using scanpy
            adata = sc.read_10x_mtx(
                temp_dir,
                var_names='gene_symbols'
            )

            # Add sample ID to obs
            adata.obs['sample'] = sample_id

            return adata

        except Exception as e:
            print(f"Error processing {sample_id}: {str(e)}")
            return None

    # Read metadata
    sra_table_path = LPath("latch://35741.account/PlotAI/SraRunTable.csv")
    metadata_df = pd.read_csv(sra_table_path.download())

    # Keep only relevant columns
    metadata_subset = metadata_df[['Individual', 'Sample Name', 'source_name', 'time', 'tissue', 'treatment']]

    # Process all GSM directories
    gsm_dir = LPath("latch://35741.account/PlotAI/GSM")
    adatas = []

    for sample_dir in gsm_dir.iterdir():
        sample_id = sample_dir.name()
        print(f"Processing {sample_id}...")

        adata = process_sample(sample_dir, sample_id)
        if adata is not None:
            # Add metadata
            sample_metadata = metadata_subset[metadata_subset['Sample Name'] == sample_id].iloc[0]
            for col in ['Individual', 'source_name', 'time', 'tissue', 'treatment']:
                adata.obs[col] = sample_metadata[col]

            adatas.append(adata)

    # Combine all samples
    if adatas:
        print(f"Combining {len(adatas)} samples...")
        combined = adatas[0].concatenate(adatas[1:], join='outer')

        # Basic preprocessing
        sc.pp.filter_cells(combined, min_genes=200)
        sc.pp.filter_genes(combined, min_cells=3)

        # Save the combined object
        combined.write_h5ad("combined_data_patients_all.h5ad")

        # Upload to Latch
        output_path = LPath("latch://35741.account/PlotAI/combined_data_patients_all.h5ad")
        output_path.upload_from("combined_data_patients_all.h5ad")

        print(f"Combined AnnData object saved and uploaded: {combined.shape[0]} cells, {combined.shape[1]} genes")
    else:
        print("No valid samples found")

import scanpy as sc
import numpy as np
import pandas as pd
from latch.ldata.path import LPath
from lplots.widgets.text import w_text_output

# Calculate QC metrics
sc.pp.calculate_qc_metrics(adata, percent_top=None, log1p=False, inplace=True)

# Calculate mitochondrial and ribosomal gene percentages
adata.var['mt'] = adata.var_names.str.startswith('MT-')
adata.var['ribo'] = adata.var_names.str.startswith(('RPS', 'RPL'))
sc.pp.calculate_qc_metrics(adata, qc_vars=['mt', 'ribo'], percent_top=None, log1p=False, inplace=True)

# Print summary statistics
summary_stats = pd.DataFrame({
    'Total genes': np.median(adata.obs.n_genes_by_counts),
    'Total counts': np.median(adata.obs.total_counts),
    'MT %': np.median(adata.obs.pct_counts_mt),
    'Ribo %': np.median(adata.obs.pct_counts_ribo)
}, index=['Median'])

w_text_output(content="### QC Metrics Summary\n" + summary_stats.to_markdown())

# Create QC violin plots with plotly
import plotly.graph_objects as go
from plotly.subplots import make_subplots

fig = make_subplots(rows=1, cols=3, subplot_titles=('Total Counts', 'Number of Genes', 'MT%'))

# Total counts violin plot
fig.add_trace(
    go.Violin(y=adata.obs['total_counts'], box_visible=True, name='Total Counts',
              points="outliers"), row=1, col=1)

# Number of genes violin plot
fig.add_trace(
    go.Violin(y=adata.obs['n_genes_by_counts'], box_visible=True, name='Num Genes',
              points="outliers"), row=1, col=2)

# MT% violin plot
fig.add_trace(
    go.Violin(y=adata.obs['pct_counts_mt'], box_visible=True, name='MT%',
              points="outliers"), row=1, col=3)

fig.update_layout(height=500, width=1200, showlegend=False)
fig

# Import necessary libraries
import scanpy as sc
import numpy as np
import pandas as pd
import plotly.express as px
from lplots.widgets.text import w_text_output

# Apply QC filters
# Filter cells based on total counts
sc.pp.filter_cells(adata, min_counts=300)

# Filter cells based on number of genes
sc.pp.filter_cells(adata, min_genes=100)

# Filter cells based on max genes and UMIs
adata = adata[adata.obs.n_genes_by_counts < 10000]
adata = adata[adata.obs.total_counts < 10000]

# Filter cells based on mitochondrial content
adata = adata[adata.obs.pct_counts_mt < 20]

# Calculate and display filtering results
filter_stats = pd.DataFrame({
    'Metric': [
        'Cells remaining',
        'Genes remaining',
        'Median UMIs per cell',
        'Median genes per cell',
        'Median MT%'
    ],
    'Value': [
        adata.shape[0],
        adata.shape[1],
        np.median(adata.obs.total_counts),
        np.median(adata.obs.n_genes_by_counts),
        np.median(adata.obs.pct_counts_mt)
    ]
})

# Create visualization of filtering results
fig = px.scatter(
    adata.obs,
    x='total_counts',
    y='n_genes_by_counts',
    color='pct_counts_mt',
    opacity=0.6,
    labels={
        'total_counts': 'Total UMIs',
        'n_genes_by_counts': 'Number of Genes',
        'pct_counts_mt': 'MT%'
    },
    title='Cell Quality Metrics After Filtering'
)

fig.update_layout(
    width=800,
    height=600,
    template='plotly_white'
)

# Display results
w_text_output(content="### Filtering Results\n" + filter_stats.to_markdown(index=False))
fig

# Import required libraries
import scanpy as sc
import numpy as np
import pandas as pd
from lplots.widgets.text import w_text_output

# Create t cell mask based on majority voting
t_cell_mask = adata.obs['majority_voting'] == 'T cells'

# Create subset of adata with only T cells
t_cell_adata = adata[t_cell_mask].copy()

# Display summary statistics
t_cell_stats = pd.DataFrame({
    'Metric': [
        'Total T cells',
        'Percent of all cells',
        'Number of genes'
    ],
    'Value': [
        t_cell_adata.shape[0],
        f"{(t_cell_adata.shape[0] / adata.shape[0] * 100):.1f}%",
        t_cell_adata.shape[1]
    ]
})

w_text_output(
    content=f"### T Cell Subset Summary\n"
    f"{t_cell_stats.to_markdown(index=False)}"
)