<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.2 20190208//EN" "http://jats.nlm.nih.gov/publishing/1.2/JATS-journalpublishing1.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="other" dtd-version="1.2" xml:lang="en">
    <front>
        <journal-meta>
            <journal-id journal-id-type="pmc">F1000Research</journal-id>
            <journal-title-group>
                <journal-title>F1000Research</journal-title>
            </journal-title-group>
            <issn pub-type="epub">2046-1402</issn>
            <publisher>
                <publisher-name>F1000 Research Limited</publisher-name>
                <publisher-loc>London, UK</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="doi">10.12688/f1000research.149494.1</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Software Tool Article</subject>
                </subj-group>
                <subj-group>
                    <subject>Articles</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>sSNAPPY: an R/Bioconductor package for single-sample directional pathway perturbation analysis</article-title>
                <fn-group content-type="pub-status">
                    <fn>
                        <p>[version 1; peer review: 2 approved with reservations]</p>
                    </fn>
                </fn-group>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Liu</surname>
                        <given-names>Wenjun</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <role content-type="http://credit.niso.org/">Visualization</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-8185-3069</uri>
                    <xref ref-type="corresp" rid="c1">a</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                    <xref ref-type="aff" rid="a2">2</xref>
                    <xref ref-type="aff" rid="a3">3</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>M&#x00e4;kinen</surname>
                        <given-names>Ville-Petteri</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a4">4</xref>
                    <xref ref-type="aff" rid="a5">5</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Tilley</surname>
                        <given-names>Wayne D</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a1">1</xref>
                </contrib>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Pederson</surname>
                        <given-names>Stephen M</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <role content-type="http://credit.niso.org/">Visualization</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0001-8197-3303</uri>
                    <xref ref-type="corresp" rid="c2">b</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                    <xref ref-type="aff" rid="a6">6</xref>
                    <xref ref-type="aff" rid="a7">7</xref>
                </contrib>
                <aff id="a1">
                    <label>1</label>Dame Roma Mitchell Cancer Research Laboratories, Adelaide Medical School, Faculty of Health and Medical Sciences, The University of Adelaide, Adelaide, South Australia, 5000, Australia</aff>
                <aff id="a2">
                    <label>2</label>Adelaide Centre for Epigenetics, School of Biomedicine, Faculty of Health and Medical Sciences, The University of Adelaide, Adelaide, South Australia, 5000, Australia</aff>
                <aff id="a3">
                    <label>3</label>The South Australian Immunogenomics Cancer Institute, Faculty of Health and Medical Sciences, The University of Adelaide, Adelaide, South Australia, 5000, Australia</aff>
                <aff id="a4">
                    <label>4</label>Computational Medicine, Faculty of Medicine, University of Oulu, Oulu, Northern Ostrobothnia, Finland</aff>
                <aff id="a5">
                    <label>5</label>Center for Life Course Health Research, Faculty of Medicine, University of Oulu, Oulu, Northern Ostrobothnia, Finland</aff>
                <aff id="a6">
                    <label>6</label>Black Ochre Data Labs, Indigenous Genomics, Telethon Kids Institute, Adelaide, South Australia, 5000, Australia</aff>
                <aff id="a7">
                    <label>7</label>John Curtin School of Medical Research, Australian National University, Canberra, Australian Capital Territory, Australia</aff>
            </contrib-group>
            <author-notes>
                <corresp id="c1">
                    <label>a</label>
                    <email xlink:href="mailto:wenjun.liu@adelaide.edu.au">wenjun.liu@adelaide.edu.au</email>
                </corresp>
                <corresp id="c2">
                    <label>b</label>
                    <email xlink:href="mailto:stephen.pederson@telethonkids.org.au">stephen.pederson@telethonkids.org.au</email>
                </corresp>
                <fn fn-type="conflict">
                    <p>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>13</day>
                <month>6</month>
                <year>2024</year>
            </pub-date>
            <pub-date pub-type="collection">
                <year>2024</year>
            </pub-date>
            <volume>13</volume>
            <elocation-id>628</elocation-id>
            <history>
                <date date-type="accepted">
                    <day>14</day>
                    <month>5</month>
                    <year>2024</year>
                </date>
            </history>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Liu W et al.</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <self-uri content-type="pdf" xlink:href="https://f1000research.com/articles/13-628/pdf"/>
            <abstract>
                <p>A common outcome of analysing RNA-Seq data is the detection of biological pathways with significantly altered activity between the conditions under investigation. Whilst many strategies test for over-representation of genes, showing changed expression within pre-defined gene-sets, these analyses typically do not account for gene-gene interactions encoded by pathway topologies, and are not able to directly predict the directional change of pathway activity. To address these issues we have developed 
                    <italic toggle="yes">sSNAPPY</italic>,now available as an R/Bioconductor package, which leverages pathway topology information to compute pathway perturbation scores and predict the direction of change across a set of pathways.</p>
                <p>Here, we demonstrate the use of 
                    <italic toggle="yes">sSNAPPY</italic> by applying the method to public scRNA-seq data, derived from ovarian cancer patient tissues collected before and after chemotherapy. Not only were we able to predict the direction of pathway perturbations discussed in the original study, but 
                    <italic toggle="yes">sSNAPPY</italic> was also able to detect significant changes of other biological processes, yielding far greater insight into the response to treatment. 
                    <italic toggle="yes">sSNAPPY</italic> represents a novel pathway analysis strategy that takes into consideration pathway topology to predict impacted biology pathways, both within related samples and across treatment groups. In addition to not relying on differentially expressed genes, the method and associated R package offers important flexibility and provides powerful visualisation tools.</p>
                <p>
                    <bold>R version</bold>: R version 4.3.3 (2024-02-29)</p>
                <p>
                    <bold>Bioconductor version</bold>: 3.18</p>
                <p>
                    <bold>Package</bold>: 1.6.1</p>
            </abstract>
            <kwd-group kwd-group-type="author">
                <kwd>RNA-Seq</kwd>
                <kwd>pathway enrichment</kwd>
                <kwd>R package</kwd>
                <kwd>topology</kwd>
                <kwd>KEGG</kwd>
                <kwd>Reactome</kwd>
                <kwd>scRNA-seq</kwd>
            </kwd-group>
            <funding-group>
                <award-group id="fund-1" xlink:href="http://dx.doi.org/10.13039/501100001026">
                    <funding-source>National Breast Cancer Foundation</funding-source>
                    <award-id>IIRS-23-069</award-id>
                </award-group>
                <award-group id="fund-2" xlink:href="http://dx.doi.org/10.13039/501100000925">
                    <funding-source>National Health and Medical Research Council</funding-source>
                    <award-id>1186647</award-id>
                </award-group>
                <funding-statement>W.D. Tilley&#x2019;s research is supported by the National Health and Medical Research Council of Australia (ID 1186647) and the National Breast Cancer Foundation Australia (ID IIRS-23-069). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript</funding-statement>
                <funding-statement>
                    <italic>The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.</italic>
                </funding-statement>
            </funding-group>
        </article-meta>
    </front>
    <body>
        <sec id="sec1" sec-type="intro">
            <title>Introduction</title>
            <p>Using pathway enrichment analysis to gain biological insights from gene expression data is a pivotal step in the analysis and interpretation of RNA-seq data, for which numerous methods have been developed (reviewed in
                <sup>
                    <xref ref-type="bibr" rid="ref1">1</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref2">2</xref>
                </sup>). Many existing methods tend to view pathways simply as a collection of gene names, as seen in those relying on the detection of differentially expressed genes and applying over-representation analysis (ORA) strategies, with alternative methods being those scoring all genes using functional class scoring (FCS), such as in Gene Set Enrichment Analysis (GSEA),
                <sup>
                    <xref ref-type="bibr" rid="ref3">3</xref>
                </sup> arguably the most widely-used approach. However, databases such as the Kyoto Encyclopaedia of Genes and Genomes (KEGG)
                <sup>
                    <xref ref-type="bibr" rid="ref4">4</xref>
                </sup> and WikiPathways
                <sup>
                    <xref ref-type="bibr" rid="ref5">5</xref>
                </sup> capture not only which genes are implicated in a certain biological process but also their interactions, activating or inhibitory roles, and their relative importance within the pathway, all of which are overlooked in ORA- and FCS-based approaches.</p>
            <p>To fully utilise this additional information, the latest generation of pathway analysis approaches include many which are topology-based such as SPIA,
                <sup>
                    <xref ref-type="bibr" rid="ref6">6</xref>
                </sup> DEGraph,
                <sup>
                    <xref ref-type="bibr" rid="ref7">7</xref>
                </sup> NetGSA
                <sup>
                    <xref ref-type="bibr" rid="ref8">8</xref>
                </sup> and PRS,
                <sup>
                    <xref ref-type="bibr" rid="ref9">9</xref>
                </sup> as well as others which explicitly model inter-gene correlations.
                <sup>
                    <xref ref-type="bibr" rid="ref10">10</xref>
                </sup> Despite differences in the null hypotheses tested across these approaches, overall, they have demonstrated enhanced sensitivity and specificity due to their abilities to take gene-gene interconnections into account.
                <sup>
                    <xref ref-type="bibr" rid="ref11">11</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref12">12</xref>
                </sup> Nevertheless, most topology-based methods focus only on comparing the activity of pathways between two treatment groups and cannot be used to score individual samples (
                <xref ref-type="fig" rid="f1">Figure 1</xref>). However, in heterogeneous data where more than one factor may be influencing observations,
                <sup>
                    <xref ref-type="bibr" rid="ref13">13</xref>
                </sup> incorporating scoring within paired samples may be desirable and may be able to reveal more nuanced insights. To address this gap, we present a Single-Sample directioNAl Pathway Perturbation analYsis methodology called 
                <ext-link ext-link-type="uri" xlink:href="https://bioconductor.org/packages/sSNAPPY">sSNAPPY</ext-link>, available as an R/Bioconductor package. This article defines how 
                <italic toggle="yes">sSNAPPY</italic> computes changes in gene expression within paired samples and propagates this through gene-set topologies, to predict the perturbation in pathway activities within paired samples, before providing summarised results across an entire dataset (
                <xref ref-type="fig" rid="f1">Figure 1</xref>). The practical usage of the 
                <italic toggle="yes">sSNAPPY</italic> R/Bioconductor package is illustrated through the analysis of a public scRNA-seq dataset using a pseudo-bulk approach, demonstrating its applicability to both bulk RNA-Seq and scRNA-Seq datasets.</p>
            <fig fig-type="figure" id="f1" orientation="portrait" position="float">
                <label>Figure 1. </label>
                <caption>
                    <title>Schematic illustration of the differences between conventional pathway analysis methods and 
                        <italic toggle="yes">sSNAPPY</italic>.</title>
                    <p>Instead of being limited to treatment-level analyses, 
                        <italic toggle="yes">sSNAPPY</italic> allows the detection of pathway perturbation within individual samples by using sample-specific estimates of fold-change instead of experiment-wide estimates. (Created with 
                        <ext-link ext-link-type="uri" xlink:href="http://BioRender.com">BioRender.com</ext-link>).</p>
                </caption>
                <graphic id="gr1" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure1.gif"/>
            </fig>
        </sec>
        <sec id="sec2" sec-type="methods">
            <title>Methods</title>
            <sec id="sec3">
                <title>Implementation</title>
                <p>
                    <italic toggle="yes">sSNAPPY</italic> is an R package that has been reviewed and published on the open-source bioinformatics software platform 
                    <ext-link ext-link-type="uri" xlink:href="https://bioconductor.org/packages/release/bioc/html/*sSNAPPY*.html">Bioconductor</ext-link> with all source code available via GitHub. The methodology itself is topology-based, designed to compute directional, single-sample, pathway perturbation scores for gene expression datasets with matched-pair, or nested designs (e.g. samples collected before and after treatment). Common examples of such designs may include treated vs control samples within cell-line passages, or across multiple treatments applied to tissue samples from within a specific donor. This allows for the detection of pathway perturbations within all samples from a treatment group, but also within individual samples, with 
                    <italic toggle="yes">sSNAPPY</italic> providing results in the form of pathway perturbation scores 1) for each set of paired samples, and 2) across all paired samples within an experimental grouping. The only data required to run 
                    <italic toggle="yes">sSNAPPY</italic>, is a log-transformed expression matrix (e.g. logCPM) with matching sample metadata describing treatment groups and the nested structure. It is assumed that all pre-processing has been performed beforehand, such as the exclusion of low-signal genes or normalisation to minimise technical artefacts like GC-bias.</p>
                <p>The first step performed by 
                    <italic toggle="yes">sSNAPPY</italic>, is to estimate sample-specific log fold-change (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                            </mml:msub>
                            <mml:mo>=</mml:mo>
                            <mml:msub>
                                <mml:mi>&#x03bc;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                            </mml:msub>
                            <mml:mo>&#x2212;</mml:mo>
                            <mml:msub>
                                <mml:mi>&#x03bc;</mml:mi>
                                <mml:mrow>
                                    <mml:mi>g</mml:mi>
                                    <mml:mn>0</mml:mn>
                                    <mml:mi>i</mml:mi>
                                </mml:mrow>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) across all genes 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>g</mml:mi>
                        </mml:math>
                    </inline-formula>, for each treatment 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>h</mml:mi>
                        </mml:math>
                    </inline-formula>, and within each set of nested replicates 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula>, by subtracting expression estimates for the baseline samples 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>&#x03bc;</mml:mi>
                                <mml:mrow>
                                    <mml:mi>g</mml:mi>
                                    <mml:mn>0</mml:mn>
                                    <mml:mi>i</mml:mi>
                                </mml:mrow>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> from those in treatment group 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>h</mml:mi>
                        </mml:math>
                    </inline-formula>. It should also be noted that 
                    <italic toggle="yes">sSNAPPY</italic> is applicable to any number of treatment/condition levels and that sample numbers within each treatment group are not required to be balanced.</p>
                <p>It is well known that in RNA-seq data, genes with lower expression tend to have greater variability in signal and more broadly spread estimates of change.
                    <sup>
                        <xref ref-type="bibr" rid="ref14">14</xref>
                    </sup> As such, we utilise a gene-level weighting strategy to down-weight fold-change estimates for low-abundance genes prior to passing these estimates to 
                    <italic toggle="yes">sSNAPPY.</italic> Gene-level weights 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>w</mml:mi>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> are obtained in a treatment-agnostic manner by fitting a loess curve through the relationship between observed gene-level variance (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03c3;</mml:mi>
                                <mml:mi>g</mml:mi>
                                <mml:mn>2</mml:mn>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula>) and average signal (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mover accent="true">
                                    <mml:mi>&#x03bc;</mml:mi>
                                    <mml:mo stretchy="true">&#x00af;</mml:mo>
                                </mml:mover>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>)(
                    <xref ref-type="fig" rid="f2">Figure 2</xref>), and taking the inverse of the loess-predicted variance as the weight 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>w</mml:mi>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                            <mml:mo>=</mml:mo>
                            <mml:mi>a</mml:mi>
                            <mml:mo>/</mml:mo>
                            <mml:mi>f</mml:mi>
                            <mml:mrow>
                                <mml:mo stretchy="true">(</mml:mo>
                                <mml:msub>
                                    <mml:mover accent="true">
                                        <mml:mi>&#x03bc;</mml:mi>
                                        <mml:mo stretchy="true">&#x00af;</mml:mo>
                                    </mml:mover>
                                    <mml:mi>g</mml:mi>
                                </mml:msub>
                                <mml:mo stretchy="true">)</mml:mo>
                            </mml:mrow>
                        </mml:math>
                    </inline-formula>, where 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>f</mml:mi>
                            <mml:mrow>
                                <mml:mo stretchy="true">(</mml:mo>
                                <mml:msub>
                                    <mml:mover accent="true">
                                        <mml:mi>&#x03bc;</mml:mi>
                                        <mml:mo stretchy="true">&#x00af;</mml:mo>
                                    </mml:mover>
                                    <mml:mi>g</mml:mi>
                                </mml:msub>
                                <mml:mo stretchy="true">)</mml:mo>
                            </mml:mrow>
                        </mml:math>
                    </inline-formula> is the predicted value from the loess curve and the constant 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>a</mml:mi>
                        </mml:math>
                    </inline-formula> ensures 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mo>&#x2211;</mml:mo>
                            <mml:msub>
                                <mml:mi>w</mml:mi>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                            <mml:mo>=</mml:mo>
                            <mml:mn>1</mml:mn>
                        </mml:math>
                    </inline-formula>. We then use these globally-weighted estimates of log fold-change (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                <mml:mo>&#x2217;</mml:mo>
                            </mml:msubsup>
                            <mml:mo>=</mml:mo>
                            <mml:msub>
                                <mml:mi>w</mml:mi>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                            <mml:msub>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) in the calculation of all subsequent pathway perturbation scores.</p>
                <fig fig-type="figure" id="f2" orientation="portrait" position="float">
                    <label>Figure 2. </label>
                    <caption>
                        <title>Gene-wise standard deviations shown against mean logCPM.</title>
                        <p>The relationship between standard deviations and expression levels is modelled by a loess fit. Whilst standard deviations are shown for the purposes of visualisation, gene-level weights are calculated using variances at this stage of the 
                            <italic toggle="yes">sSNAPPY</italic> algorithm.</p>
                    </caption>
                    <graphic id="gr2" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure2.gif"/>
                </fig>
                <p>
                    <italic toggle="yes">sSNAPPY</italic> extends the topology-based scoring algorithm initially proposed in 
                    <italic toggle="yes">SPIA</italic>
                    <sup>
                        <xref ref-type="bibr" rid="ref6">6</xref>
                    </sup> which propagates fold-change estimates from genes considered as differentially expressed through pathway topologies, to compute a perturbation score for each pathway. In contrast to 
                    <italic toggle="yes">SPIA</italic>, which relies on a defined set of differentially expressed genes, 
                    <italic toggle="yes">sSNAPPY</italic> uses fold-change estimates from all detected genes. By modifying the algorithm to incorporate single-sample, weighted estimates of fold-change, we are able to numerically represent changes in a pathway within a given sample, and subsequently model these across all samples within a treatment group. Thus, we define the single-sample perturbation score (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>S</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) for a given pathway 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>p</mml:mi>
                        </mml:math>
                    </inline-formula> and treatment 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>h</mml:mi>
                        </mml:math>
                    </inline-formula> for a set of nested samples 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula>:
                    <disp-formula id="e1">
                        <mml:math display="block">
                            <mml:mtable columnalign="left" displaystyle="true">
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:msub>
                                            <mml:mi>S</mml:mi>
                                            <mml:mi mathvariant="italic">hip</mml:mi>
                                        </mml:msub>
                                        <mml:mo>=</mml:mo>
                                        <mml:munder>
                                            <mml:mo>&#x2211;</mml:mo>
                                            <mml:mrow>
                                                <mml:mi>g</mml:mi>
                                                <mml:mo>&#x2208;</mml:mo>
                                                <mml:msub>
                                                    <mml:mi>G</mml:mi>
                                                    <mml:mi>p</mml:mi>
                                                </mml:msub>
                                            </mml:mrow>
                                        </mml:munder>
                                        <mml:mrow>
                                            <mml:mo stretchy="true">[</mml:mo>
                                            <mml:msub>
                                                <mml:mi>S</mml:mi>
                                                <mml:mtext mathvariant="italic">ghip</mml:mtext>
                                            </mml:msub>
                                            <mml:mo>&#x2212;</mml:mo>
                                            <mml:msubsup>
                                                <mml:mi>&#x03b4;</mml:mi>
                                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                                <mml:mo>&#x2217;</mml:mo>
                                            </mml:msubsup>
                                            <mml:mo stretchy="true">]</mml:mo>
                                        </mml:mrow>
                                        <mml:mo>,</mml:mo>
                                        <mml:mtext>where</mml:mtext>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:msub>
                                            <mml:mi>S</mml:mi>
                                            <mml:mtext mathvariant="italic">ghip</mml:mtext>
                                        </mml:msub>
                                        <mml:mo>=</mml:mo>
                                        <mml:msubsup>
                                            <mml:mi>&#x03b4;</mml:mi>
                                            <mml:mi mathvariant="italic">ghi</mml:mi>
                                            <mml:mo>&#x2217;</mml:mo>
                                        </mml:msubsup>
                                        <mml:mo>+</mml:mo>
                                        <mml:munder>
                                            <mml:mo>&#x2211;</mml:mo>
                                            <mml:mrow>
                                                <mml:mi>g</mml:mi>
                                                <mml:mo>&#x2032;</mml:mo>
                                                <mml:mo>&#x2208;</mml:mo>
                                                <mml:msub>
                                                    <mml:mi>U</mml:mi>
                                                    <mml:mi mathvariant="italic">gp</mml:mi>
                                                </mml:msub>
                                            </mml:mrow>
                                        </mml:munder>
                                        <mml:msub>
                                            <mml:mi>&#x03b2;</mml:mi>
                                            <mml:mrow>
                                                <mml:mi mathvariant="italic">gg</mml:mi>
                                                <mml:mo>&#x2032;</mml:mo>
                                                <mml:mi>p</mml:mi>
                                            </mml:mrow>
                                        </mml:msub>
                                        <mml:mfrac>
                                            <mml:msub>
                                                <mml:mi>S</mml:mi>
                                                <mml:mrow>
                                                    <mml:mi>g</mml:mi>
                                                    <mml:mo>&#x2032;</mml:mo>
                                                    <mml:mi mathvariant="italic">hip</mml:mi>
                                                </mml:mrow>
                                            </mml:msub>
                                            <mml:msub>
                                                <mml:mi>N</mml:mi>
                                                <mml:mrow>
                                                    <mml:mi>g</mml:mi>
                                                    <mml:mo>&#x2032;</mml:mo>
                                                    <mml:mi>p</mml:mi>
                                                </mml:mrow>
                                            </mml:msub>
                                        </mml:mfrac>
                                    </mml:mtd>
                                </mml:mtr>
                            </mml:mtable>
                        </mml:math>
                    </disp-formula>and:
                    <list list-type="bullet">
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>G</mml:mi>
                                            <mml:mi>p</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> represents the set of genes in pathway 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>p</mml:mi>
                                    </mml:math>
                                </inline-formula>, such that 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                        <mml:mo>&#x2208;</mml:mo>
                                        <mml:msub>
                                            <mml:mi>G</mml:mi>
                                            <mml:mi>p</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula>
                            </p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>S</mml:mi>
                                            <mml:mtext mathvariant="italic">ghip</mml:mtext>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the gene-, treatment- and sample-specific perturbation score for pathway 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>p</mml:mi>
                                    </mml:math>
                                </inline-formula>
                            </p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msubsup>
                                            <mml:mi>&#x03b4;</mml:mi>
                                            <mml:mi mathvariant="italic">ghi</mml:mi>
                                            <mml:mo>&#x2217;</mml:mo>
                                        </mml:msubsup>
                                        <mml:mo>=</mml:mo>
                                        <mml:msub>
                                            <mml:mi>w</mml:mi>
                                            <mml:mi>g</mml:mi>
                                        </mml:msub>
                                        <mml:msub>
                                            <mml:mi>&#x03b4;</mml:mi>
                                            <mml:mi mathvariant="italic">ghi</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the weighted log fold-change of gene 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                    </mml:math>
                                </inline-formula> as described above</p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>U</mml:mi>
                                            <mml:mi mathvariant="italic">gp</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the subset of 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>G</mml:mi>
                                            <mml:mi>p</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> containing only the genes directly upstream of gene 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                    </mml:math>
                                </inline-formula>, and not including gene 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                    </mml:math>
                                </inline-formula>
                            </p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>&#x03b2;</mml:mi>
                                            <mml:mrow>
                                                <mml:mi mathvariant="italic">gg</mml:mi>
                                                <mml:mo>&#x2032;</mml:mo>
                                                <mml:mi>p</mml:mi>
                                            </mml:mrow>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the pair-wise gene-gene interactions
                                <sup>
                                    <xref ref-type="bibr" rid="ref6">6</xref>
                                </sup> encoded by the topology matrix for genes 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                    </mml:math>
                                </inline-formula> and 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msup>
                                            <mml:mi>g</mml:mi>
                                            <mml:mo>&#x2032;</mml:mo>
                                        </mml:msup>
                                        <mml:mo>&#x2208;</mml:mo>
                                        <mml:msub>
                                            <mml:mi>U</mml:mi>
                                            <mml:mi mathvariant="italic">gp</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula>
                            </p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>N</mml:mi>
                                            <mml:mi mathvariant="italic">gp</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the number of downstream genes from any gene 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>g</mml:mi>
                                    </mml:math>
                                </inline-formula>
                            </p>
                        </list-item>
                        <list-item>
                            <label>&#x2022;</label>
                            <p>
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:msub>
                                            <mml:mi>S</mml:mi>
                                            <mml:mi mathvariant="italic">hip</mml:mi>
                                        </mml:msub>
                                    </mml:math>
                                </inline-formula> is the accumulated pathway perturbation score for pathway 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>p</mml:mi>
                                    </mml:math>
                                </inline-formula> in treatment 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>h</mml:mi>
                                    </mml:math>
                                </inline-formula> within sample 
                                <inline-formula>
                                    <mml:math display="inline">
                                        <mml:mi>i</mml:mi>
                                    </mml:math>
                                </inline-formula> across all genes in the pathway</p>
                        </list-item>
                    </list>
                </p>
                <p>To scale single-sample pathway perturbation scores (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>S</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) so they are comparable across pathways, and to test for significance of individual scores, null distributions of perturbation scores for each pathway are generated through a sample permutation strategy, which retains any existing correlation structures between genes within a pathway. During permutation, all sample labels are randomly shuffled and permuted pseudo-pairs formed from the re-shuffled labels. Single-sample fold-changes are then calculated for each pseudo-pair of permuted samples while the rest of the scoring algorithm remains unchanged. The median and median absolute deviation (MAD) are calculated from the set of permuted perturbation scores within each pathway and used to normalise the raw perturbation scores to robust 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>Z</mml:mi>
                        </mml:math>
                    </inline-formula>-scores, i.e. 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>Z</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>.</p>
                <p>All possible permuted pseudo-pairs are sampled unless otherwise specified, such that in an experiment with 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>I</mml:mi>
                        </mml:math>
                    </inline-formula> total samples, the maximum number of unique permuted pairs is 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>P</mml:mi>
                            <mml:mo>=</mml:mo>
                            <mml:mfrac>
                                <mml:mrow>
                                    <mml:mi>I</mml:mi>
                                    <mml:mo>!</mml:mo>
                                </mml:mrow>
                                <mml:mrow>
                                    <mml:mrow>
                                        <mml:mo stretchy="true">(</mml:mo>
                                        <mml:mi>I</mml:mi>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>2</mml:mn>
                                        <mml:mo stretchy="true">)</mml:mo>
                                    </mml:mrow>
                                    <mml:mo>!</mml:mo>
                                </mml:mrow>
                            </mml:mfrac>
                            <mml:mo>=</mml:mo>
                            <mml:mi>I</mml:mi>
                            <mml:mo>&#x00d7;</mml:mo>
                            <mml:mrow>
                                <mml:mo stretchy="true">(</mml:mo>
                                <mml:mi>I</mml:mi>
                                <mml:mo>&#x2212;</mml:mo>
                                <mml:mn>1</mml:mn>
                                <mml:mo stretchy="true">)</mml:mo>
                            </mml:mrow>
                        </mml:math>
                    </inline-formula>. Permutation 
                    <italic toggle="yes">p</italic>-values are calculated for each 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>S</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> value, indicating the approximate significance of pathway perturbation at the single-sample level. Relying on symmetry around zero, these are derived by assessing the proportion of permuted scores with absolute values as extreme, or more extreme, than the absolute value of test perturbation score within each pathway.
                    <sup>
                        <xref ref-type="bibr" rid="ref15">15</xref>
                    </sup> Since the smallest achievable permutation 
                    <italic toggle="yes">p</italic>-value is 1/
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>P</mml:mi>
                        </mml:math>
                    </inline-formula>, accurate estimation of small 
                    <italic toggle="yes">p</italic>-value requires a large number of permutations that is only feasible in data with a large sample size. As a guideline, GSEA recommends a minimum of 7 samples in each treatment group for utilizing their phenotype permutation approach,
                    <sup>
                        <xref ref-type="bibr" rid="ref16">16</xref>
                    </sup> and under 
                    <italic toggle="yes">sSNAPPY</italic> this would yield 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>P</mml:mi>
                        </mml:math>
                    </inline-formula> = 182 unique permuted pseudo-pairs and minimal permutation 
                    <italic toggle="yes">p</italic>-values of 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>p</mml:mi>
                            <mml:mo>&#x2265;</mml:mo>
                            <mml:mn>0.0055</mml:mn>
                            <mml:mspace width="0.25em"/>
                        </mml:math>
                    </inline-formula>at each individual sample and pathway-level.</p>
                <p>Apart from assessing whether a pathway&#x2019;s activity is changed significantly within an individual sample, an important question may be the detection of changes in pathway activity across all samples within a treatment. This can be performed by modelling 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>Z</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> values using regression models and incorporating Smyth&#x2019;s moderated 
                    <italic toggle="yes">t</italic>-statistic
                    <sup>
                        <xref ref-type="bibr" rid="ref17">17</xref>
                    </sup> as implemented in 
                    <italic toggle="yes">limma.</italic>
                    <sup>
                        <xref ref-type="bibr" rid="ref18">18</xref>
                    </sup> The single-sample nature of 
                    <italic toggle="yes">sSNAPPY</italic>&#x2019;s pathway perturbation scores is particularly helpful for datasets with complex experimental designs or known confounding factors as these can also be incorporated into the final regression models.</p>
                <p>The Bioconductor package 
                    <italic toggle="yes">graphite</italic>
                    <sup>
                        <xref ref-type="bibr" rid="ref19">19</xref>
                    </sup> provides functions that can be used to retrieve pathway topologies from a database and convert topology information to adjacency matrices. To streamline this process, we have implemented a convenience function, where users only need to provide the name of the desired database and species to retrieve all topology information in the format required by the scoring algorithm with the correct type of gene identifiers (i.e. EntrezID). Importantly, we noted that 
                    <italic toggle="yes">graphite</italic>
                    <sup>
                        <xref ref-type="bibr" rid="ref19">19</xref>
                    </sup> retrieves topology matrices in varied orientations from different databases; in KEGG pathways, columns represent downstream relationships, whereas in Reactome and WikiPathway, columns indicate upstream regulation. However, this discrepancy is not taken account of in the 
                    <monospace>runSPIA()</monospace> function of graphite. We resolve the issue by transposing the topology matrices retrieved from the Reactome and WikiPathway as part of our convenience function.</p>
            </sec>
            <sec id="sec4">
                <title>Operation</title>
                <p>The package has been tested on all operating systems, requiring R &gt; 4.3.0, and can be installed using BiocManager as follows.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">if</styled-content>
                            </bold> (
                            <bold>
                                <styled-content style="color:#8F5A03">!</styled-content>
                                <styled-content style="color:#214A88">requireNamespace</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"BiocManager"</styled-content>, 
                            <styled-content style="color:#214A88">quietly</styled-content> = 
                            <styled-content style="color:#8F5A03">TRUE</styled-content>))</monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">install.packages</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"BiocManager"</styled-content>)</monospace>

                        <monospace>BiocManager
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">install</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"sSNAPPY"</styled-content>)</monospace>
                    </preformat>
                </p>
            </sec>
        </sec>
        <sec id="sec5">
            <title>Use cases</title>
            <sec id="sec6">
                <title>Required packages</title>
                <p>The set of additional required packages for this workflow can be installed using

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>pkg 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#6F9A04">"tidyverse"</styled-content>, 
                            <styled-content style="color:#6F9A04">"patchwork"</styled-content>, 
                            <styled-content style="color:#6F9A04">"kableExtra"</styled-content>, 
                            <styled-content style="color:#6F9A04">"AnnotationHub"</styled-content>, 
                            <styled-content style="color:#6F9A04">"edgeR"</styled-content>, 
                            <styled-content style="color:#6F9A04">"here"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#6F9A04">"pander"</styled-content>, 
                            <styled-content style="color:#6F9A04">"colorspace"</styled-content>, 
                            <styled-content style="color:#6F9A04">"SPIA"</styled-content>, 
                            <styled-content style="color:#6F9A04">"fgsea"</styled-content>
                        </monospace>

                        <monospace>)</monospace>

                        <monospace>BiocManager
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">install</styled-content>
                            </bold>(pkg, 
                            <styled-content style="color:#214A88">update</styled-content> = 
                            <styled-content style="color:#8F5A03">FALSE</styled-content>, 
                            <styled-content style="color:#214A88">force</styled-content> = 
                            <styled-content style="color:#8F5A03">FALSE</styled-content>, 
                            <styled-content style="color:#214A88">ask</styled-content> = 
                            <styled-content style="color:#8F5A03">FALSE</styled-content>)</monospace>
                    </preformat>
                </p>
                <p>Once installed, the complete set of packages can be loaded into the session.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(sSNAPPY)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(tidyverse)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(magrittr)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(patchwork)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(kableExtra)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(AnnotationHub)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(edgeR)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(pander)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(colorspace)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(SPIA)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">library</styled-content>
                            </bold>(fgsea)</monospace>
                    </preformat>
                </p>
            </sec>
            <sec id="sec7">
                <title>Retrieval of pathway topology matrices</title>
                <p>Retrieving pathway topology information from a chosen database is the next key step required during preparation for running 
                    <italic toggle="yes">sSNAPPY</italic>, and this is the only step requiring internet access. If running on an HPC cluster where internet access may be restricted, this step can be performed separately with topologies saved by the user as an RDS object. Using the Reactome database
                    <sup>
                        <xref ref-type="bibr" rid="ref20">20</xref>
                    </sup> in this workflow, the retrieved topology information will be stored as a list where each element corresponds to a pathway and the numbers in the matrices encode gene-gene interactions.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>gsTopology 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">retrieve_topology</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">database</styled-content> = 
                            <styled-content style="color:#6F9A04">"reactome"</styled-content>, 
                            <styled-content style="color:#214A88">species</styled-content> = 
                            <styled-content style="color:#6F9A04">"hsapiens"</styled-content>)</monospace>
                    </preformat>
                </p>
                <p>In addition to downloading topology matrices for all pathways, it is also possible to provide a restricted set of keywords for a targeted analysis. For example, passing the argument 
                    <monospace>keyword = c("metabolism", "estrogen")</monospace> would only return the subset of pathways which match either of these keywords. Multiple databases are also able to be searched by passing a vector of database names to the 
                    <monospace>database</monospace> argument.</p>
            </sec>
            <sec id="sec8">
                <title>Data</title>
                <p>To demonstrate the application of 
                    <italic toggle="yes">sSNAPPY</italic>, we used pre-processed counts from a publicly available scRNA-seq dataset, retrieved from Gene Expression Omnibus (GEO) with accession code GSE165897. This dataset consists of 11 high-grade serous ovarian cancer (HGSOC) patient samples taken before and after chemotherapy.
                    <sup>
                        <xref ref-type="bibr" rid="ref21">21</xref>
                    </sup> 
                    <italic toggle="yes">sSNAPPY</italic> was used to re-analyse data from the epithelial cells as they were the primary focus of the original study. Since 
                    <italic toggle="yes">sSNAPPY</italic> was designed primarily for bulk RNA-seq data, counts from epithelial cells within the same samples were first summed into pseudo-bulk profiles, giving rise to a total of 22 samples. We considered a gene detectable if we observed &gt;1.5 counts per million in 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mo>&#x2265;</mml:mo>
                        </mml:math>
                    </inline-formula> 11 of the 22 samples, ideally representing all samples from a complete treatment group. 11,101 (33.8%) of the 32,847 annotated genes passed this selection criteria and were included for downstream analyses. Conditional quantile normalisation
                    <sup>
                        <xref ref-type="bibr" rid="ref22">22</xref>
                    </sup> was then applied to mitigate potential biases introduced by gene length and GC content. The normalised logCPM matrix of the processed dataset and sample metadata can be downloaded from 
                    <ext-link ext-link-type="uri" xlink:href="https://github.com/Wenjun-Liu/F1000_sSNAPPY_manuscript/tree/master/data">here</ext-link>.</p>
                <p>To begin running the 
                    <italic toggle="yes">sSNAPPY</italic> workflow, we first load our expression matrix and define our sample-level metadata. Importantly, the row names of the expression matrix must be specified as EntrezGene IDs, for compatibility with pathway databases. Genes without EntrezGene IDs were excluded during pre-processing, leaving 10,098 genes in the example expression matrix. The treatment column within our metadata is expected to be a factor, with the reference level interpreted as the control treatment.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>readr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                            </bold>
                            <bold>
                                <styled-content style="color:#214A88">local_edition</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">1</styled-content>)</monospace>

                        <monospace>logCPM 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">read_tsv</styled-content>
                            </bold>(here
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">here</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"data/logCPM.tsv"</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">column_to_rownames</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"entrezid"</styled-content>)</monospace>

                        <monospace>sample_meta 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">read_tsv</styled-content>
                            </bold>(here
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">here</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"data/sample_meta.tsv"</styled-content>), 
                            <styled-content style="color:#214A88">col_types =</styled-content> 
                            <styled-content style="color:#6F9A04">"cfccncnc"</styled-content>)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">head</styled-content>
                            </bold>(sample_meta)</monospace>


                        <monospace>## # A tibble: 6 &#x00d7; 8</monospace>

                        <monospace>##   sample         treatment  patient_id anatomical_location Age Stage PFI CRS</monospace>

                        <monospace>##   &lt;chr&gt;           &lt;fct&gt;       &lt;chr&gt;      &lt;chr&gt;             &lt;dbl&gt; &lt;chr&gt; &lt;dbl&gt; &lt;chr&gt;</monospace>

                        <monospace>## 1 EOC372_treat &#x2026; treatmen &#x2026; EOC372     Peritoneum         68 IIIC    460 1</monospace>

                        <monospace>## 2 EOC372_post- &#x2026; post-NACT  EOC372     Peritoneum         68 IIIC    460 1</monospace>

                        <monospace>## 3 EOC443_post- &#x2026; post-NACT  EOC443     Omentum            54 IVA     177 3</monospace>

                        <monospace>## 4 EOC443_treat &#x2026; treatmen &#x2026; EOC443     Omentum            54 IVA     177 3</monospace>

                        <monospace>## 5 EOC540_treat &#x2026; treatmen &#x2026; EOC540     Omentum            62 IIIC    126 2</monospace>

                        <monospace>## 6 EOC540_post- &#x2026; post-NACT  EOC540     Omentum            62 IIIC    126 2</monospace>
                    </preformat>
                </p>
            </sec>
            <sec id="sec9">
                <title>Score single-sample pathway perturbation</title>
                <p>To compute the single-sample fold-changes (i.e. logFC) required for the set of perturbation scores, samples must be &#x2018;matched pairs&#x2019; or nested, as discussed earlier, and treatments performed within each patient represents the nesting structure in this dataset. The factor (patient_id) defining the nested structure is passed to 
                    <monospace>weight_ss_fc()</monospace> using the 
                    <monospace>groupBy</monospace> parameter.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>weightedFC 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">weight_ss_fc</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as.matrix</styled-content>
                            </bold> (logCPM), 
                            <styled-content style="color:#214A88">metadata =</styled-content> sample_meta,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">sampleColumn =</styled-content> 
                            <styled-content style="color:#6F9A04">"sample"</styled-content>, 
                            <styled-content style="color:#214A88">groupBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"patient_id"</styled-content>, 
                            <styled-content style="color:#214A88">treatColumn =</styled-content> 
                            <styled-content style="color:#6F9A04">"treatment"</styled-content>
                        </monospace>

                        <monospace>)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">glimpse</styled-content>
                            </bold> (weightedFC)</monospace>
                    </preformat>
                </p>
                <p>The output of 
                    <monospace>weight_ss_fc()</monospace> is a 
                    <monospace>list</monospace> where one element is the matrix of weighted single-sample fold-changes (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                <mml:mo>&#x2217;</mml:mo>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula>), with rows corresponding to genes and columns to samples, and the other element is the vector of gene-wise weights (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>w</mml:mi>
                                <mml:mi>g</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) used to calculate the weighted log fold-change (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                <mml:mo>&#x2217;</mml:mo>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula>). By default, the string 
                    <monospace>ENTREZID:</monospace> is added to all row names of the 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                <mml:mo>&#x2217;</mml:mo>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula> matrix to be compatible with the format Reactome pathway topologies are retrieved in.</p>
                <p>The matrix of 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msubsup>
                                <mml:mi>&#x03b4;</mml:mi>
                                <mml:mi mathvariant="italic">ghi</mml:mi>
                                <mml:mo>&#x2217;</mml:mo>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula> values is then passed to pathway topologies to compute gene-wise perturbation scores for all genes within a pathway, before being summed into a single score for each pathway. The function 
                    <monospace>raw_gene_pert()</monospace> returns an initial list, with each element containing the gene-level perturbation scores for a given pathway. These matrices are also able to be used during downstream analysis to identify which genes play the most significant roles in each pathway, as demonstrated in later sections. Pathway-level perturbation scores (
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>S</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>) are then returned as a 
                    <monospace>data.frame</monospace> containing sample and gene-set names after calling 
                    <monospace>pathway_pert()</monospace>. Pathways with zero perturbation scores across all genes and samples are automatically dropped at this step. In addition, pathways with aggregated pathway-level scores of zero in all samples will also be dropped by default, unless otherwise specified through the 
                    <monospace>drop</monospace> parameter of the 
                    <monospace>pathway_pert()</monospace> function.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>genePertScore 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">raw_gene_pert</styled-content>
                            </bold>(weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weighted_logFC, gsTopology)</monospace>

                        <monospace>ssPertScore 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">pathway_pert</styled-content>
                            </bold>(genePertScore, weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weighted_logFC)</monospace>

                        <monospace>ssPertScore 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">
                                <bold>split</bold>
                            </styled-content>(
                            <styled-content style="color:#214A88">f =</styled-content> .
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>gs_name) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>    
                            <bold>
                                <styled-content style="color:#214A88">keep</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#8F5A03">~</styled-content>
                                <styled-content style="color:#214A88">all</styled-content>
                            </bold>(. 
                            <bold>
                                <styled-content style="color:#8F5A03">!=</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">0</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>    
                            <bold>
                                <styled-content style="color:#214A88">length</styled-content>
                            </bold>()</monospace>

                        <monospace>## [1] 1093</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">head</styled-content>
                            </bold> (ssPertScore)</monospace>

                        <monospace>##             sample         score                           gs_name</monospace>

                        <monospace>## 1 EOC372_post-NACT -2.292688e-04 reactome. Interleukin-6 signaling</monospace>

                        <monospace>## 2 EOC443_post-NACT -2.447003e-04 reactome. Interleukin-6 signaling</monospace>

                        <monospace>## 3 EOC540_post-NACT -1.848758e-04 reactome. Interleukin-6 signaling</monospace>

                        <monospace>## 4   EOC3_post-NACT -1.229489e-04 reactome. Interleukin-6 signaling</monospace>

                        <monospace>## 5  EOC87_post-NACT  3.427132e-05 reactome. Interleukin-6 signaling</monospace>

                        <monospace>## 6 EOC136_post-NACT  2.822155e-04 reactome. Interleukin-6 signaling</monospace>
                    </preformat>
                </p>
            </sec>
            <sec id="sec10">
                <title>Sample permutation for normalisation and significance testing</title>
                <p>The range of 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>S</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> values obtained from the complete set of pathways will vary greatly, due to the variability in topology structures. To determine the significance of individual scores and transform scores to ensure they are comparable across pathways, 
                    <italic toggle="yes">sSNAPPY</italic> utilises a sample-permutation strategy to estimate the null distributions of perturbation scores and derive 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>Z</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> scores. Since sample labels will be permuted randomly to put samples into pseudo-pairs, sample metadata is not required by the 
                    <monospace>generate_permuted_scores</monospace> function. All possible random pairs between samples will be sampled unless otherwise specified. In this example dataset with a total of 22 samples, the full set of 462 (i.e. 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>P</mml:mi>
                            <mml:mo>=</mml:mo>
                            <mml:mn>22</mml:mn>
                            <mml:mo>&#x00d7;</mml:mo>
                            <mml:mn>21</mml:mn>
                        </mml:math>
                    </inline-formula>) permuted scores will be computed for each pathway.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>permutedScore 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">generate_permuted_scores</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as.matrix</styled-content>
                            </bold> (logCPM),</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsTopology =</styled-content> gsTopology[
                            <bold>
                                <styled-content style="color:#214A88">names</styled-content>
                            </bold>(genePertScore)],</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">weight =</styled-content> weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weight</monospace>

                        <monospace>)</monospace>
                    </preformat>
                </p>
                <p>Apart from pathways whose permuted perturbation scores are consistently zero, and which will be dropped by default, the empirical distributions of remaining pathways are expected to be approximately normally distributed with mean 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>&#x03bc;</mml:mi>
                            <mml:mo>=</mml:mo>
                            <mml:mn>0</mml:mn>
                        </mml:math>
                    </inline-formula>, but with the scale of distributions heavily impacted by both the number of genes within each pathway and the overall topology. To demonstrate this, we randomly selected 6 pathways to demonstrate their quantile-quantile (q-q) plot and visualised the distributions of their permuted perturbation scores as boxplots (
                    <xref ref-type="fig" rid="f3">Figure 3</xref>).

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">set.seed</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">123</styled-content>)</monospace>

                        <monospace>random_pathways 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> permutedScore 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">keep</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#8F5A03">~</styled-content>
                                <styled-content style="color:#214A88">all</styled-content>
                            </bold>(. 
                            <bold>
                                <styled-content style="color:#8F5A03">!=</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">0</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  .[
                            <bold>
                                <styled-content style="color:#214A88">sample</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">seq_along</styled-content>
                            </bold>(.), 
                            <styled-content style="color:#0000CF">6</styled-content>)] 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as.data.frame</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">pivot_longer</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">cols =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">everything</styled-content>
                            </bold>(), 
                            <styled-content style="color:#214A88">names_to =</styled-content> 
                            <styled-content style="color:#6F9A04">"gs_name"</styled-content>, 
                            <styled-content style="color:#214A88">values_to =</styled-content> 
                            <styled-content style="color:#6F9A04">"score"</styled-content>
                        </monospace>

                        <monospace>  ) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">mutate</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">gs_name =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">str_replace_all</styled-content>
                            </bold>(gs_name, 
                            <styled-content style="color:#6F9A04">"</styled-content>
                            <bold>
                                <styled-content style="color:#8F5A03">\\</styled-content>
                            </bold>
                            <styled-content style="color:#6F9A04">."</styled-content>, 
                            <styled-content style="color:#6F9A04">" "</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">gs_name =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">str_remove_all</styled-content>
                            </bold>(gs_name, 
                            <styled-content style="color:#6F9A04">"reactome "</styled-content>)</monospace>

                        <monospace>  )</monospace>

                        <monospace>p1 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> random_pathways 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">ggplot</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">aes</styled-content>
                            </bold> (
                            <styled-content style="color:#214A88">sample =</styled-content> score, 
                            <styled-content style="color:#214A88">colour =</styled-content> gs_name)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">stat_qq</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">stat_qq_line</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">colour =</styled-content> 
                            <styled-content style="color:#6F9A04">"black"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">facet_wrap</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#8F5A03">~</styled-content>
                                <styled-content style="color:#214A88">str_wrap</styled-content>
                            </bold>(gs_name, 
                            <styled-content style="color:#214A88">width =</styled-content> 
                            <styled-content style="color:#0000CF">25</styled-content>), 
                            <styled-content style="color:#214A88">scales=</styled-content> 
                            <styled-content style="color:#6F9A04">"free"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">labs</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">y =</styled-content> 
                            <styled-content style="color:#6F9A04">"Permuted Perturbation Score"</styled-content>, 
                            <styled-content style="color:#214A88">x =</styled-content> 
                            <styled-content style="color:#6F9A04">"Theoretical Quantiles"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_bw</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position =</styled-content> 
                            <styled-content style="color:#6F9A04">"none"</styled-content>,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">14</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">strip.text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">16</styled-content>))</monospace>

                        <monospace>p2 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> random_pathways 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">ggplot</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">aes</styled-content>
                            </bold> (gs_name, score, 
                            <styled-content style="color:#214A88">fill =</styled-content> gs_name)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">geom_boxplot</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_x_discrete</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">labels =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">function</styled-content>
                            </bold>(x) 
                            <bold>
                                <styled-content style="color:#214A88">str_wrap</styled-content>
                            </bold>(x, 
                            <styled-content style="color:#214A88">width =</styled-content> 
                            <styled-content style="color:#0000CF">10</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_fill_discrete</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">name =</styled-content> 
                            <styled-content style="color:#6F9A04">"Gene-set Name"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">labs</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">x =</styled-content> 
                            <styled-content style="color:#6F9A04">"Pathway"</styled-content>, 
                            <styled-content style="color:#214A88">y =</styled-content> 
                            <styled-content style="color:#6F9A04">"Permuted Perturbation Score"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_bw</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position =</styled-content> 
                            <styled-content style="color:#6F9A04">"none"</styled-content>,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">axis.title =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">16</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">axis.text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">14</styled-content>)</monospace>

                        <monospace>  )</monospace>

                        <monospace>(p1
                            <bold>
                                <styled-content style="color:#8F5A03">/</styled-content>
                            </bold>p2) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">plot_annotation</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">tag_levels =</styled-content> 
                            <styled-content style="color:#6F9A04">"A"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">plot_layout</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">heights =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0.6</styled-content>, 
                            <styled-content style="color:#0000CF">0.4</styled-content>))</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f3" orientation="portrait" position="float">
                    <label>Figure 3. </label>
                    <caption>
                        <title>(A) Q-Q plot and (B) distributions of permuted perturbation scores of six randomly selected pathways.</title>
                        <p>All sampled empirical distributions are approximately normally distributed with a mean of zero.</p>
                    </caption>
                    <graphic id="gr3" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure3.gif"/>
                </fig>
                <p>The distributions obtained from label permutations are then used to convert each pathway-level score into the robust 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>Z</mml:mi>
                                <mml:mi mathvariant="italic">hij</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>-score using the function 
                    <monospace>normalise_by_permu()</monospace>. Two-sided 
                    <italic toggle="yes">p</italic>-values for individual scores are computed based on how extreme test scores are in comparison to permuted scores for each pathway and corrected for multiple testing using any of the available methods, returning the FDR-adjusted values by default. In our example data, no pathways would be considered as significantly perturbed at the single-sample level using an FDR adjustment with 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>&#x03b1;</mml:mi>
                        </mml:math>
                    </inline-formula> = 0.05.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>normalisedScores 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">normalise_by_permu</styled-content>
                            </bold>(permutedScore, ssPertScore,</monospace>

                        <monospace>                                       
                            <styled-content style="color:#214A88">sortBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"pvalue"</styled-content>)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">head</styled-content>
                            </bold> (normalisedScores)</monospace>


                        <monospace>##               MAD MEDIAN</monospace>

                        <monospace>## 2306 0.0006067519      0</monospace>

                        <monospace>## 2525 0.0002909911      0</monospace>

                        <monospace>## 5869 0.0001652241      0</monospace>

                        <monospace>## 5871 0.0001652241      0</monospace>

                        <monospace>## 5872 0.0001652241      0</monospace>

                        <monospace>## 7721 0.0275863198      0</monospace>

                        <monospace>##                                                           gs_name</monospace>

                        <monospace>## 2306 reactome.Golgi Cisternae Pericentriolar Stack Reorganization</monospace>

                        <monospace>## 2525       reactome.DNA Damage/Telomere Stress Induced Senescence</monospace>

                        <monospace>## 5869                        reactome.Defective CHST6 causes MCDC1</monospace>

                        <monospace>## 5871           reactome.Defective ST3GAL3 causes MCT12 and EIEE15</monospace>

                        <monospace>## 5872       reactome.Defective B4GALT1 causes B4GALT1-CDG (CDG-2d)</monospace>

                        <monospace>## 7721                        reactome.Mitochondrial protein import</monospace>

                        <monospace>##                sample         score   robustZ      pvalue adjPvalue</monospace>

                        <monospace>## 2306 EOC153_post-NACT -0.0013632057 -2.246727 0.004329004 1.0000000</monospace>

                        <monospace>## 2525 EOC153_post-NACT  0.0006149637  2.113342 0.004329004 1.0000000</monospace>

                        <monospace>## 5869 EOC349_post-NACT -0.0003304944 -2.000279 0.004329004 0.9993248</monospace>

                        <monospace>## 5871 EOC349_post-NACT -0.0003304944 -2.000279 0.004329004 0.9993248</monospace>

                        <monospace>## 5872 EOC349_post-NACT -0.0003304944 -2.000279 0.004329004 0.9993248</monospace>

                        <monospace>## 7721 EOC443_post-NACT -0.0598366717 -2.169070 0.004329004 0.9471861</monospace>
                    </preformat>
                </p>
                <p>A key question of interest in our example dataset is to identify which biological processes were impacted by chemotherapy across the entire group of patients. Using the sample-level output obtained above, we can explore this by applying t-tests or regression models across all samples. In order to minimise spurious results, Smyth&#x2019;s moderated t-statistics
                    <sup>
                        <xref ref-type="bibr" rid="ref17">17</xref>
                    </sup> can be applied across the complete dataset, with a constant variance assumed across all pathways, given that we are using Z-scores. To perform this analysis, 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>Z</mml:mi>
                                <mml:mi mathvariant="italic">hip</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> values were converted to a matrix and standard 
                    <italic toggle="yes">limma</italic> methodologies were used. For our use case here, where only one treatment group is present, no design matrix is required, and a simple t-test is appropriate.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>z_matrix 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> normalisedScores 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">select</styled-content>
                            </bold> (robustZ, gs_name, sample) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">pivot_wider</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">names_from =</styled-content> 
                            <styled-content style="color:#6F9A04">"sample"</styled-content>, 
                            <styled-content style="color:#214A88">values_from =</styled-content> 
                            <styled-content style="color:#6F9A04">"robustZ"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">column_to_rownames</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"gs_name"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as.matrix</styled-content>
                            </bold>()</monospace>

                        <monospace>z_fits 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> z_matrix 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">lmFit</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">design =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">rep</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">1</styled-content>, 
                            <bold>
                                <styled-content style="color:#214A88">ncol</styled-content>
                            </bold>(.))) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">eBayes</styled-content>
                            </bold>()</monospace>

                        <monospace>top_table 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">topTable</styled-content>
                            </bold>(z_fits, 
                            <styled-content style="color:#214A88">number =</styled-content> 
                            <styled-content style="color:#8F5A03">Inf</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as_tibble</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">rownames =</styled-content> 
                            <styled-content style="color:#6F9A04">"gs_name"</styled-content>)</monospace>

                        <monospace>sigPathway 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">filter</styled-content>
                            </bold> (top_table, adj.P.Val 
                            <bold>
                                <styled-content style="color:#8F5A03">&lt;</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">0.05</styled-content>)</monospace>
                    </preformat>
                </p>
                <p>121 out of the 1094 tested Reactome pathways have an FDR &lt; 0.05 in the moderated t-test, and were considered to be significantly perturbed at the group level. 
                    <xref ref-type="table" rid="T1">Table 1</xref> presents the top 10 significantly inhibited and activated pathways, along with their predicted direction of change.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>table 1 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> sigPathway 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">mutate</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">Direction =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">ifelse</styled-content>
                            </bold> (logFC 
                            <bold>
                                <styled-content style="color:#8F5A03">&lt;</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">0</styled-content>, 
                            <styled-content style="color:#6F9A04">"Inhibited"</styled-content>, 
                            <styled-content style="color:#6F9A04">"Activated"</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">gs_name =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">str_remove_all</styled-content>
                            </bold>(gs_name, 
                            <styled-content style="color:#6F9A04">"reactome."</styled-content>)</monospace>

                        <monospace>  ) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">split</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">f =</styled-content> .
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>Direction) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">lapply</styled-content>
                            </bold> (dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                            </bold>slice, 
                            <styled-content style="color:#0000CF">1</styled-content>
                            <bold>
                                <styled-content style="color:#8F5A03">:</styled-content>
                            </bold>
                            <styled-content style="color:#0000CF">10</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">bind_rows</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">select</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">Pathway =</styled-content> gs_name, 
                            <styled-content style="color:#214A88">Change =</styled-content> logFC, P.Value, 
                            <styled-content style="color:#214A88">FDR =</styled-content> adj.P.Val, Direction</monospace>

                        <monospace>  )</monospace>
                    </preformat>
                </p>
                <table-wrap id="T1" orientation="portrait" position="float">
                    <label>Table 1. </label>
                    <caption>
                        <title>Significantly impacted Reactome pathways identified among post-chemotherapy samples using 
                            <italic toggle="yes">sSNAPPY</italic>. Only the 10 most significantly inhibited and 10 most significantly activated pathways are shown.</title>
                    </caption>
                    <table content-type="article-table" frame="hsides">
                        <thead>
                            <tr>
                                <th align="left" colspan="1" rowspan="1" valign="top">Pathway</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">Change</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">P.Value</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">FDR</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">Direction</th>
                            </tr>
                        </thead>
                        <tbody>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Signaling by Retinoic Acid</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.601</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">4.45E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Regulation of pyruvate dehydrogenase (PDH) complex</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.598</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">4.59E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Pyruvate metabolism</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.598</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">4.59E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Pyruvate metabolism and Citric Acid (TCA) cycle</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.598</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">4.59E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Negative regulation of MAPK pathway</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.627</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">5.40E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0174</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Translocation of ZAP-70 to Immunological synapse</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.624</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">8.75E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0218</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Generation of second messenger molecules</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.624</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">8.75E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0218</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">MHC class II antigen presentation</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.628</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0012</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0241</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">NGF-stimulated transcription</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.582</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0014</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0247</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Downstream TCR signaling</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.633</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0014</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0247</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Activated</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Interleukin-35 Signalling</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.902</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.69E-05</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0151</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Sphingolipid de novo biosynthesis</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.896</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">2.75E-05</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0151</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">SUMOylation of DNA replication proteins</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.819</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">6.86E-05</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Condensation of Prometaphase Chromosomes</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.904</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">9.91E-05</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Epigenetic regulation of gene expression</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.790</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.21E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Polo-like kinase mediated events</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.879</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.43E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Chromatin modifying enzymes</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.790</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.52E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">Chromatin organization</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.790</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.52E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">SUMO E3 ligases SUMOylate target proteins</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.767</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">1.83E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">APC-Cdc20 mediated degradation of Nek2A</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">-0.838</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">2.13E-04</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">0.0152</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">Inhibited</td>
                            </tr>
                        </tbody>
                    </table>
                </table-wrap>
                <p>For enrichment analysis in the original study,
                    <sup>
                        <xref ref-type="bibr" rid="ref21">21</xref>
                    </sup> unsupervised clustering was performed on all cells labelled as cancer cells. Clusters were then annotated manually by performing pathway enrichment testing on cluster marker genes. Two clusters, associated with proliferative DNA repair signatures and stress-related markers, contained significantly higher numbers of post-chemotherapy cells than pre-treatment ones.
                    <sup>
                        <xref ref-type="bibr" rid="ref21">21</xref>
                    </sup> The representative pathways enriched in the stress-associated cluster were 
                    <italic toggle="yes">IL6-mediated signaling events</italic>, 
                    <italic toggle="yes">TNF signaling pathway</italic>, and 
                    <italic toggle="yes">cellular responses to stress</italic>, and the other post-chemotherapy cell dominated cluster in the original study was enriched for pathways associated with cell proliferation and DNA repair, such as the Cell cycle, DNA repair, Homology directed repair (HDR) through homologous recombination, and the Fanconi anaemia pathway.</p>
                <p>Whilst the published enrichment analysis was performed using ConsensusPathDB,
                    <sup>
                        <xref ref-type="bibr" rid="ref23">23</xref>
                    </sup> in order to use pathway topologies we chose the Reactome set of pathways.
                    <sup>
                        <xref ref-type="bibr" rid="ref20">20</xref>
                    </sup> 
                    <italic toggle="yes">sSNAPPY</italic> not only detected many significant perturbed pathways that are highly concordant with the pathways reported to be enriched in the original study but also includes an expected direction of change in activity. For example, the DNT repair pathway SUMOylation of DNA damage response and repair proteins pathway was predicted to be significantly inhibited by chemotherapy. The single-sample nature of the 
                    <italic toggle="yes">sSNAPPY</italic> output also provides great flexibility: apart from considering all treated samples as biological replicates, users may elect to perform an analysis incorporating other phenotypic traits which may impact a patient&#x2019;s responses to chemotherapy, such as disease stages or tumour grades. To perform this step using the moderated t-statistic strategy and extend the above analysis, an appropriate design matrix is the only additional requirement for model-fitting, or alternatively, samples may be subset as may be appropriate.</p>
            </sec>
            <sec id="sec11">
                <title>Visualising perturbed pathways as networks</title>
                <p>A valuable feature of 
                    <italic toggle="yes">sSNAPPY</italic> is the provision of several visualisation functions to assist in the presentation and interpretation of results. Biological pathways are not independent of each other with many genes playing a role across multiple pathways, and as such, viewing pathway analysis results as a network can be a powerful way to intuitively summarise the results and facilitate interpretation of the underlying biology. The 
                    <monospace>plot_gs_network()</monospace> function allows users to easily convert a list of relevant biological pathways to a network where edges between pathway nodes represent overlapping genes. Defined by the colorBy parameter, pathway nodes can be coloured by either the predicted direction of change or by significance levels (
                    <xref ref-type="fig" rid="f4">Figure 4</xref>). The returned plot is a 
                    <italic toggle="yes">ggplot2</italic>
                    <sup>
                        <xref ref-type="bibr" rid="ref24">24</xref>
                    </sup> object, meaning that components of the plotting theme and other parameters can be customized as for any other ggplot2 objects.</p>
                <p>In the following example, we&#x2019;ll inspect the 10 most significantly inhibited and 10 most significantly activated pathways, which involved four steps to prepare the data: 1) rename the logFC column to reflect the true meaning of the value and, 2) create a categorical variable with the pathway status, 3) transform p-values for simpler visualisation and 4) obtain a subset of pathways to visualise.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>sigPathway 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> sigPathway 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">rename</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">Z =</styled-content> logFC) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">mutate</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">status =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">ifelse</styled-content>
                            </bold>(Z 
                            <bold>
                                <styled-content style="color:#8F5A03">&gt;</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">0</styled-content>, 
                            <styled-content style="color:#6F9A04">"Activated"</styled-content>, 
                            <styled-content style="color:#6F9A04">"Inhibited"</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">status =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">as.factor</styled-content>
                            </bold>(status),</monospace>

                        <monospace>    
                            <styled-content style="color:#6F9A04">'</styled-content>
                            <styled-content style="color:#214A88">-log10(p)</styled-content>
                            <styled-content style="color:#6F9A04">'</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <bold>
                                <styled-content style="color:#8F5A03">-</styled-content>
                                <styled-content style="color:#214A88">log10</styled-content>
                            </bold>(P.Value)</monospace>

                        <monospace>  ) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">split</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">f =</styled-content> .
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>status) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">lapply</styled-content>
                            </bold>(dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                            </bold>slice, 
                            <styled-content style="color:#0000CF">1</styled-content>
                            <bold>
                                <styled-content style="color:#8F5A03">:</styled-content>
                            </bold>
                            <styled-content style="color:#0000CF">10</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">bind_rows</styled-content>
                            </bold>()</monospace>

                        <monospace>
                            <styled-content style="color:#8F5A03">
                                <italic toggle="yes"># Plot the network structure</italic>
                            </styled-content>
                        </monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">set.seed</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">150</styled-content>)</monospace>

                        <monospace>p1 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">plot_gs_network</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">normalisedScores =</styled-content> sigPathway, 
                            <styled-content style="color:#214A88">gsTopology =</styled-content> gsTopology, 
                            <styled-content style="color:#214A88">colorBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"status"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsNameSize =</styled-content> 
                            <styled-content style="color:#0000CF">3</styled-content>
                        </monospace>

                        <monospace>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_colour_manual</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">values =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"red"</styled-content>, 
                            <styled-content style="color:#6F9A04">"blue"</styled-content>, 
                            <styled-content style="color:#6F9A04">"grey30"</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_void</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">10</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position =</styled-content> 
                            <styled-content style="color:#6F9A04">"inside"</styled-content>,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position.inside =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0.95</styled-content>, 
                            <styled-content style="color:#0000CF">0.95</styled-content>)</monospace>

                        <monospace>  )</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">set.seed</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">150</styled-content>)</monospace>

                        <monospace>p2 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">plot_gs_network</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">normalisedScores =</styled-content> sigPathway,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsTopology =</styled-content> gsTopology,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">colorBy=</styled-content> 
                            <styled-content style="color:#6F9A04">"-log10(p)"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsNameSize =</styled-content> 
                            <styled-content style="color:#0000CF">3</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsLegTitle =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">expression</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">paste</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#8F5A03">-</styled-content>
                            </bold>log[
                            <styled-content style="color:#0000CF">10</styled-content>], 
                            <styled-content style="color:#6F9A04">"p"</styled-content>))</monospace>

                        <monospace>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_colour_viridis_c</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_void</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">8</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.title =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">10</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position =</styled-content> 
                            <styled-content style="color:#6F9A04">"inside"</styled-content>,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.position.inside =</styled-content> 
                            <bold>c</bold>(
                            <styled-content style="color:#0000CF">0.95</styled-content>, 
                            <styled-content style="color:#0000CF">0.95</styled-content>)</monospace>

                        <monospace>  )</monospace>

                        <monospace>p1
                            <bold>
                                <styled-content style="color:#8F5A03">/</styled-content>
                            </bold>p2 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content> 
                                <styled-content style="color:#214A88">plot_annotation</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">tag_levels =</styled-content> 
                            <styled-content style="color:#6F9A04">"A"</styled-content>)</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f4" orientation="portrait" position="float">
                    <label>Figure 4. </label>
                    <caption>
                        <title>Significantly perturbed Reactome pathways identified among post-chemotherapy samples using 
                            <italic toggle="yes">sSNAPPY</italic>.</title>
                        <p>Each node represents a significantly perturbed Reactome pathway, with nodes coloured by (A) predicted direction of change and (B) -log10(p). The 10 most significantly inhibited and 10 most significantly activated pathways are shown.</p>
                    </caption>
                    <graphic id="gr4" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure4.gif"/>
                </fig>
                <p>An advantage of visualising pathway analysis results using network structures is that it allows the identification of highly connected pathways (
                    <xref ref-type="fig" rid="f4">Figure 4</xref>). To summarise related pathways and further enable interpretation, we can apply community detection
                    <sup>
                        <xref ref-type="bibr" rid="ref25">25</xref>
                    </sup> to group related pathways into &#x2018;communities&#x2019;. 
                    <italic toggle="yes">sSNAPPY</italic>&#x2019;s 
                    <monospace>plot_community()</monospace> function is a &#x201c;one-stop shop&#x201d; for applying a community detection algorithm of the user&#x2019;s choice to the network structure and annotating identified communities by the most common pathway category, denoting the main biological processes perturbed in that community. The most recent categories for both KEGG and Reactome databases were curated from their respective website (
                    <ext-link ext-link-type="uri" xlink:href="https://www.genome.jp/kegg/pathway.html">KEGG website</ext-link> &amp; 
                    <ext-link ext-link-type="uri" xlink:href="https://reactome.org/PathwayBrowser/#/R-HSA-1632852&amp;PATH=R-HSA-9612973">Reactome website</ext-link>) and included as parts of 
                    <italic toggle="yes">sSNAPPY.</italic> Analyses involving other pathway databases may require user-provided pathway categories. When the information about pathway categorisations is available, annotation of pathway communities is automatically completed. In the current dataset, the Louvain method was applied to the network of biological pathways and revealed five primary communities: 1) Adaptive Immune System; 2) Cell Cycle, Mitotic; 3) Chromatin modifying enzymes &amp; Epigenetic regulation of gene expression; 4) Post-translational protein modification and 5) The citric acid (TCA) cycle and respiratory electron transport (
                    <xref ref-type="fig" rid="f5">Figure 5</xref>). The largest community formed was the Adaptive Immune System pathway, indicating a clear immune-signalling aspect to these results.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">load</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">system.file</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"extdata"</styled-content>, 
                            <styled-content style="color:#6F9A04">"gsAnnotation_df_reactome.rda"</styled-content>, 
                            <styled-content style="color:#214A88">package =</styled-content> 
                            <styled-content style="color:#6F9A04">"sSNAPPY"</styled-content>))</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">set.seed</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">199</styled-content>)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">plot_community</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">normalisedScores =</styled-content> sigPathway,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">gsTopology =</styled-content> gsTopology,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">gsAnnotation =</styled-content> gsAnnotation_df_reactome,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">colorBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"status"</styled-content>,</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">lb_size =</styled-content> 
                            <styled-content style="color:#0000CF">3</styled-content>
                        </monospace>

                        <monospace>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_colour_manual</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">values =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"red"</styled-content>, 
                            <styled-content style="color:#6F9A04">"blue"</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_fill_viridis_d</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_x_continuous</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">expand =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">expansion</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0.2</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_y_continuous</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">expand =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">expansion</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0.2</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">guides</styled-content>
                            </bold> (
                            <styled-content style="color:#214A88">fill =</styled-content> 
                            <styled-content style="color:#6F9A04">"none"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_void</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>      
                            <styled-content style="color:#214A88">legend.position =</styled-content> 
                            <styled-content style="color:#6F9A04">"inside"</styled-content>,</monospace>

                        <monospace>      
                            <styled-content style="color:#214A88">legend.position.inside =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0.9</styled-content>, 
                            <styled-content style="color:#0000CF">0.1</styled-content>)</monospace>

                        <monospace>  )</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f5" orientation="portrait" position="float">
                    <label>Figure 5. </label>
                    <caption>
                        <title>Community structures among significantly perturbed Reactome pathways that were identified among post-chemotherapy samples using 
                            <italic toggle="yes">sSNAPPY</italic>.</title>
                        <p>The Louvain algorithm was applied to detect community structures, revealing biological processes associated with highly ranked pathways. The top 20 which were the most perturbed by chemotherapy are shown.</p>
                    </caption>
                    <graphic id="gr5" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure5.gif"/>
                </fig>
                <p>A key advantage of 
                    <italic toggle="yes">sSNAPPY</italic> is that it does not require the prior identification of differentially expressed genes, as this is a common challenge faced within clinical datasets. However, knowing which genes are implicated in the perturbation of pathways, particularly those which influence multiple pathways, can provide valuable insights for hypothesis generation and the underlying biological mechanisms. Therefore, 
                    <italic toggle="yes">sSNAPPY</italic> provides another visualisation feature called 
                    <monospace>plot_gs2gene</monospace>, which enables the inclusion of select genes from each pathway using network structures. Users can provide a vector of fold-change estimates to visualise genes within pathways, showing their estimated change in expression. As pathways often include hundreds of genes, it is recommended to filter for genes most likely to be playing a significant role. In this example dataset, only genes within the top 500 when ranking by the magnitude of the mean ssFC were included (
                    <xref ref-type="fig" rid="f6">Figure 6</xref>). An alternative strategy will be to select genes based on test-statistics, however, this decision is up to the individual researcher.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>meanFC 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">rowMeans</styled-content>
                            </bold>(weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weighted_logFC)
                            <bold>
                                <styled-content style="color:#8F5A03">/</styled-content>
                            </bold>weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weight</monospace>

                        <monospace>top500 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">rank</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">1</styled-content>
                            <bold>
                                <styled-content style="color:#8F5A03">/</styled-content>
                                <styled-content style="color:#214A88">abs</styled-content>
                            </bold> (meanFC)) 
                            <bold>
                                <styled-content style="color:#8F5A03">&lt;=</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">500</styled-content>
                        </monospace>
                    </preformat>
                </p>
                <p>Since Reactome pathway topologies were retrieved using EntrezIDs, users can provide a data.frame mapping EntrezIDs to their chosen identifiers, such as gene names, through the mapEntrezID parameter, in order to make the visualisations more informative. A 
                    <monospace>data.frame</monospace> converting EntrezIDs to Ensembl gene names was derived from the Ensembl Release 101
                    <sup>
                        <xref ref-type="bibr" rid="ref26">26</xref>
                    </sup> and has been made available as part of the package and serves as a helpful template for future mapping operations by users.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">load</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">system.file</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"extdata"</styled-content>, 
                            <styled-content style="color:#6F9A04">"entrez2name.rda"</styled-content>, 
                            <styled-content style="color:#214A88">package =</styled-content> 
                            <styled-content style="color:#6F9A04">"sSNAPPY"</styled-content>))</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">head</styled-content>
                            </bold>(entrez2name)</monospace>


                        <monospace>## # A tibble: 6 &#x00d7; 2</monospace>

                        <monospace>##   entrezid           mapTo</monospace>

                        <monospace>##   &lt;chr&gt;              &lt;chr&gt;</monospace>

                        <monospace>## 1 ENTREZID:84771     DDX11L1</monospace>

                        <monospace>## 2 ENTREZID:727856    DDX11L1/DDX11L9/DDX11L10</monospace>

                        <monospace>## 3 ENTREZID:100287102 DDX11L1</monospace>

                        <monospace>## 4 ENTREZID:100287596 DDX11L1/DDX11L9</monospace>

                        <monospace>## 5 ENTREZID:102725121 DDX11L1</monospace>

                        <monospace>## 6 ENTREZID:653635    WASH7P</monospace>


                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">set.seed</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">195</styled-content>)</monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">plot_gs2gene</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">normalisedScores =</styled-content> sigPathway,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsTopology =</styled-content> gsTopology,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">colorGsBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"status"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">mapEntrezID =</styled-content> entrez2name,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">geneFC =</styled-content> meanFC[top500],</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">layout =</styled-content> 
                            <styled-content style="color:#6F9A04">"kk"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">edgeAlpha =</styled-content> 
                            <styled-content style="color:#0000CF">1</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsNodeSize =</styled-content> 
                            <styled-content style="color:#0000CF">5</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">geneNodeSize =</styled-content> 
                            <styled-content style="color:#0000CF">3</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">gsNameSize =</styled-content> 
                            <styled-content style="color:#0000CF">4</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">geneNameSize =</styled-content> 
                            <styled-content style="color:#0000CF">3.5</styled-content>
                        </monospace>

                        <monospace>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_colour_gradient2</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">name =</styled-content> 
                            <styled-content style="color:#6F9A04">"Mean ssFC"</styled-content>, 
                            <styled-content style="color:#214A88">low =</styled-content> 
                            <styled-content style="color:#6F9A04">"orange"</styled-content>, 
                            <styled-content style="color:#214A88">high =</styled-content> 
                            <styled-content style="color:#6F9A04">"green4"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">scale_fill_manual</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">values =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"red"</styled-content>, 
                            <styled-content style="color:#6F9A04">"blue"</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_void</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.text =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">12</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">legend.title =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">element_text</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">size =</styled-content> 
                            <styled-content style="color:#0000CF">12</styled-content>)</monospace>

                        <monospace>  )</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f6" orientation="portrait" position="float">
                    <label>Figure 6. </label>
                    <caption>
                        <title>Genes associated with significantly perturbed Reactome pathways that were identified among post-chemotherapy samples using 
                            <italic toggle="yes">sSNAPPY</italic>.</title>
                        <p>Significantly perturbed Reactome pathways identified among post-chemotherapy samples using 
                            <italic toggle="yes">sSNAPPY</italic>, showing any genes in the top 500 ranked by magnitude of change in expression, and which pathways they are likely contributing to. Only the 10 most significantly inhibited and 10 most significantly activated pathways are shown.</p>
                    </caption>
                    <graphic id="gr6" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure6.gif"/>
                </fig>
            </sec>
            <sec id="sec12">
                <title>Identifying key gene contributions</title>
                <p>To further investigate a specific pathway and elucidate which are the key genes contributing to the final perturbation score, we can generate a heatmap via 
                    <monospace>plot_gene_contribution()</monospace> which shows the gene-level perturbation scores for the top-ranked members of a given pathway. This function takes advantage of the plotting capabilities of the 
                    <italic toggle="yes">
                        <monospace>pheatmap</monospace>
                    </italic> package,
                    <sup>
                        <xref ref-type="bibr" rid="ref27">27</xref>
                    </sup> and as such, other annotations are also able to be easily included, such as patient response, or which general ranges the pathway-level normalised Z-Scores are in. Inclusion of the Z-Scores enabled the assessment of the level of perturbation predicted in each sample and key genes involved (
                    <xref ref-type="fig" rid="f7">Figure 7</xref>).

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>
                            <styled-content style="color:#214A88">path_name</styled-content> 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <styled-content style="color:#6F9A04">"SUMOylation of DNA replication proteins"</styled-content>
                        </monospace>

                        <monospace>z_breaks 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">round</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">qnorm</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">0</styled-content>, 
                            <styled-content style="color:#0000CF">0.025</styled-content>, 
                            <styled-content style="color:#0000CF">0.05</styled-content>, 
                            <styled-content style="color:#0000CF">0.25</styled-content>, 
                            <styled-content style="color:#0000CF">0.75</styled-content>, 
                            <styled-content style="color:#0000CF">0.95</styled-content>, 
                            <styled-content style="color:#0000CF">0.975</styled-content>, 
                            <styled-content style="color:#0000CF">1</styled-content>)), 
                            <styled-content style="color:#0000CF">2</styled-content>)</monospace>

                        <monospace>annotation_df 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> normalisedScores 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">filter</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">str_detect</styled-content>
                            </bold>(gs_name, path_name)) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">left_join</styled-content>
                            </bold>(dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">select</styled-content>
                            </bold>(sample_meta, sample, CRS), 
                            <styled-content style="color:#214A88">by =</styled-content> 
                            <styled-content style="color:#6F9A04">"sample"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">mutate</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#6F9A04">'</styled-content>
                            <styled-content style="color:#214A88">Z Range</styled-content>
                            <styled-content style="color:#6F9A04">'</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">cut</styled-content>
                            </bold>(robustZ, 
                            <styled-content style="color:#214A88">breaks =</styled-content> z_breaks, 
                            <styled-content style="color:#214A88">include.lowest =</styled-content> 
                            <styled-content style="color:#8F5A03">TRUE</styled-content>),</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">sample =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">str_remove_all</styled-content>
                            </bold>(sample, 
                            <styled-content style="color:#6F9A04">"_post-NACT"</styled-content>)</monospace>

                        <monospace>  ) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">select</styled-content>
                            </bold>(sample, 
                            <styled-content style="color:#6F9A04">'</styled-content>
                            <styled-content style="color:#214A88">Z Range</styled-content>
                            <styled-content style="color:#6F9A04">'</styled-content>, CRS)</monospace>

                        <monospace>z_levels 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">levels</styled-content>
                            </bold> (annotation_df
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>
                            <styled-content style="color:#6F9A04">'</styled-content>
                            <styled-content style="color:#214A88">Z Range</styled-content>
                            <styled-content style="color:#6F9A04">'</styled-content>)</monospace>

                        <monospace>annotation_col 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">list</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">CRS =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"3"</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <styled-content style="color:#6F9A04">"#4B0055"</styled-content>, 
                            <styled-content style="color:#6F9A04">"2"</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <styled-content style="color:#6F9A04">"#009B95"</styled-content>, 
                            <styled-content style="color:#6F9A04">"1"</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <styled-content style="color:#6F9A04">"#FDE333"</styled-content>),</monospace>

                        <monospace>  
                            <styled-content style="color:#6F9A04">'</styled-content>
                            <styled-content style="color:#214A88">Z Range</styled-content>
                            <styled-content style="color:#6F9A04">'</styled-content> 
                            <styled-content style="color:#8F5A03">=</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">setNames</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <bold>
                                <styled-content style="color:#214A88">colorRampPalette</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"navyblue"</styled-content>, 
                            <styled-content style="color:#6F9A04">"white"</styled-content>, 
                            <styled-content style="color:#6F9A04">"darkred"</styled-content>))(
                            <bold>
                                <styled-content style="color:#214A88">length</styled-content>
                            </bold>(z_levels)),</monospace>

                        <monospace>    z_levels</monospace>

                        <monospace>  )</monospace>

                        <monospace>)</monospace>

                        <monospace>mat 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> genePertScore 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  .[[
                            <bold>
                                <styled-content style="color:#214A88">which</styled-content>
                            </bold> (
                            <bold>
                                <styled-content style="color:#214A88">str_detect</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">names</styled-content>
                            </bold>(.), path_name))]] 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">set_colnames</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">str_remove_all</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">colnames</styled-content>
                            </bold>(.), 
                            <styled-content style="color:#6F9A04">"_post-NACT"</styled-content>)) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  .[
                            <bold>
                                <styled-content style="color:#214A88">rownames</styled-content>
                            </bold>(.) 
                            <bold>
                                <styled-content style="color:#8F5A03">%in%</styled-content> 
                                <styled-content style="color:#214A88">rownames</styled-content>
                            </bold> (weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weighted_logFC),]</monospace>

                        <monospace>max_pert 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">max</styled-content>
                            </bold> (
                            <bold>
                                <styled-content style="color:#214A88">abs</styled-content>
                            </bold> (
                            <bold>
                                <styled-content style="color:#214A88">range</styled-content>
                            </bold> (mat))) 
                            <bold>
                                <styled-content style="color:#8F5A03">*</styled-content>
                            </bold> 
                            <styled-content style="color:#0000CF">1.01</styled-content>
                        </monospace>

                        <monospace>
                            <bold>
                                <styled-content style="color:#214A88">plot_gene_contribution</styled-content>
                            </bold>(</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">genePertMatr =</styled-content> mat,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">color =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">rev</styled-content>
                            </bold> (colorspace
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">divergex_hcl</styled-content>
                            </bold>(
                            <styled-content style="color:#0000CF">100</styled-content>, 
                            <styled-content style="color:#214A88">palette =</styled-content> 
                            <styled-content style="color:#6F9A04">"RdBu"</styled-content>)),</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">breaks =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">seq</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#8F5A03">-</styled-content>
                            </bold>max_pert, max_pert, 
                            <styled-content style="color:#214A88">length.out =</styled-content> 
                            <styled-content style="color:#0000CF">100</styled-content>),</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">annotation_df =</styled-content>annotation_df,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">annotation_colors =</styled-content> annotation_col,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">filterBy =</styled-content> 
                            <styled-content style="color:#6F9A04">"mean"</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">mapEntrezID =</styled-content> entrez2name,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">cutree_rows =</styled-content> 
                            <styled-content style="color:#0000CF">2</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">cutree_cols =</styled-content> 
                            <styled-content style="color:#0000CF">2</styled-content>,</monospace>

                        <monospace>  
                            <styled-content style="color:#214A88">main =</styled-content> 
                            <bold>
                                <styled-content style="color:#214A88">paste</styled-content>
                            </bold> (path_name, 
                            <styled-content style="color:#6F9A04">"[REACTOME]"</styled-content>)</monospace>

                        <monospace>)</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f7" orientation="portrait" position="float">
                    <label>Figure 7. </label>
                    <caption>
                        <title>Gene-level perturbation scores for the top 10 genes in "SUMOylation of DNA replication proteins" pathway.</title>
                        <p>All pathway genes were ranked by average contribution to the perturbation score to select the top 10 genes. Samples were annotated by patient chemotherapy response score (CRS), along with the range for sample-level Z-scores as a guide to sample-specific pathway perturbation. The genes 
                            <italic toggle="yes">CDCA8</italic>, 
                            <italic toggle="yes">TOP2A</italic>, 
                            <italic toggle="yes">UBE2I</italic>, 
                            <italic toggle="yes">BIRC5</italic> were identified as possible key drivers of inhibition for this pathway.</p>
                    </caption>
                    <graphic id="gr7" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure7.gif"/>
                </fig>
                <p>From this heatmap we can identify four candidate genes which are likely to be making a contribution to the inhibition of the SUMOylation of DNA replication proteins pathway upon chemotherapy, such as 
                    <italic toggle="yes">CDCA8</italic>, 
                    <italic toggle="yes">TOP2A</italic>, 
                    <italic toggle="yes">UBE2I</italic>, 
                    <italic toggle="yes">BIRC5</italic> (
                    <xref ref-type="fig" rid="f7">Figure 7</xref>). These four genes are all associated with tumour progression and invasiveness and have been studied in the context of ovarian cancer. Both ubiquitin conjugating enzyme E2I (
                    <italic toggle="yes">UBE2I</italic>) and cell division cycle associated 8 (
                    <italic toggle="yes">CDCA8</italic>) genes have been identified as oncogenes in numerous cancer types, including ovarian cancer.
                    <sup>
                        <xref ref-type="bibr" rid="ref28">28</xref>
                    </sup>
                    <sup>,</sup>
                    <sup>
                        <xref ref-type="bibr" rid="ref29">29</xref>
                    </sup> Notably, in ovarian cancer, elevated 
                    <italic toggle="yes">UBE2I</italic> expression has been associated with poorer clinical outcomes.
                    <sup>
                        <xref ref-type="bibr" rid="ref30">30</xref>
                    </sup> Similarly, 
                    <italic toggle="yes">BIRC5</italic> encodes a protein which is also a predictor of inferior ovarian cancer patient outcome.
                    <sup>
                        <xref ref-type="bibr" rid="ref31">31</xref>
                    </sup> Lastly, 
                    <italic toggle="yes">Topoisomerase II</italic>
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>&#x03b1;</mml:mi>
                        </mml:math>
                    </inline-formula> (
                    <italic toggle="yes">TOP2A</italic>), which encodes DNA topoisomerase, has been identified as a gene that promotes the tumorigenesis of HGSOC tumours.
                    <sup>
                        <xref ref-type="bibr" rid="ref32">32</xref>
                    </sup> Aligning with the report by Chekerov et al.
                    <sup>
                        <xref ref-type="bibr" rid="ref33">33</xref>
                    </sup> that expression of 
                    <italic toggle="yes">TOP2A</italic> in ovarian tumour cells decreases as a response to chemotherapy,
                    <sup>
                        <xref ref-type="bibr" rid="ref33">33</xref>
                    </sup> the median single-sample logFC of 
                    <italic toggle="yes">TOP2A</italic> was negative among the HGSOC post-chemotherapy samples included in this study (
                    <xref ref-type="fig" rid="f8">Figure 8</xref>). The other three selected potential driver genes (
                    <italic toggle="yes">CDCA8</italic>, 
                    <italic toggle="yes">UBE2I</italic>, and 
                    <italic toggle="yes">BIRC5</italic>) also had negative median single-sample logFC in post-chemotherapy samples (
                    <xref ref-type="fig" rid="f8">Figure 8</xref>). Considering the implication of these four genes in ovarian cancer, decreases in their expression after chemotherapy treatment potentially indicate a favorable response to therapy.</p>
                <p>By annotating the heatmap of gene-wise perturbation scores with patient chemotherapy response score (CRS), we also noted that the strongest inhibition of the SUMOylation of DNA replication proteins pathway was in the patient with the highest CRS score of 3 (i.e sample EOC443). CRS is an indicator of the relative length of progression-free survival after chemotherapy, where a score of 3 represents the longest survival. Hence inhibition of the SUMOylation of DNA replication proteins pathway might mediate favorable response to chemotherapy in ovarian cancer patients. We acknowledge that our analysis was limited to a small number of patients, which restricts the generalizability of the results. However, despite this limitation, these findings underscore the strength of 
                    <italic toggle="yes">sSNAPPY</italic> as a valuable tool for hypothesis generation not otherwise possible. Not only can 
                    <italic toggle="yes">sSNAPPY</italic> predict directional pathway perturbations, but it also enables the identification of potential driver genes which are strongly associated with these perturbations.

                    <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
                        <monospace>gene2plot 
                            <styled-content style="color:#8F5A03">&lt;-</styled-content> entrez2name 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  dplyr
                            <bold>
                                <styled-content style="color:#8F5A03">::</styled-content>
                                <styled-content style="color:#214A88">filter</styled-content>
                            </bold>(mapTo 
                            <bold>
                                <styled-content style="color:#8F5A03">%in%</styled-content> 
                                <styled-content style="color:#214A88">c</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"CDCA8"</styled-content>, 
                            <styled-content style="color:#6F9A04">"TOP2A"</styled-content>, 
                            <styled-content style="color:#6F9A04">"UBE2I"</styled-content>, 
                            <styled-content style="color:#6F9A04">"BIRC5"</styled-content>))</monospace>

                        <monospace>(weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weighted_logFC
                            <bold>
                                <styled-content style="color:#8F5A03">/</styled-content>
                            </bold>weightedFC
                            <bold>
                                <styled-content style="color:#8F5A03">$</styled-content>
                            </bold>weight) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">as.data.frame</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">rownames_to_column</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"entrezid"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">pivot_longer</styled-content>
                            </bold>(</monospace>

                        <monospace>    
                            <styled-content style="color:#214A88">cols =</styled-content> 
                            <bold>
                                <styled-content style="color:#8F5A03">-</styled-content>
                                <styled-content style="color:#214A88">all_of</styled-content>
                            </bold>(
                            <styled-content style="color:#6F9A04">"entrezid"</styled-content>), 
                            <styled-content style="color:#214A88">names_to =</styled-content> 
                            <styled-content style="color:#6F9A04">"sample"</styled-content>, 
                            <styled-content style="color:#214A88">values_to =</styled-content> 
                            <styled-content style="color:#6F9A04">"ssFC"</styled-content>
                        </monospace>

                        <monospace>  ) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">inner_join</styled-content>
                            </bold>(gene2plot, 
                            <styled-content style="color:#214A88">by =</styled-content> 
                            <styled-content style="color:#6F9A04">"entrezid"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">%&gt;%</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">ggplot</styled-content>
                            </bold>(
                            <bold>
                                <styled-content style="color:#214A88">aes</styled-content>
                            </bold>(mapTo, ssFC, 
                            <styled-content style="color:#214A88">fill =</styled-content> mapTo)) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">geom_boxplot</styled-content>
                            </bold>() 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">labs</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">x =</styled-content> 
                            <styled-content style="color:#6F9A04">""</styled-content>, 
                            <styled-content style="color:#214A88">fill =</styled-content> 
                            <styled-content style="color:#6F9A04">"Gene"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">geom_hline</styled-content>
                            </bold>(
                            <styled-content style="color:#214A88">yintercept =</styled-content> 
                            <styled-content style="color:#0000CF">0</styled-content>, 
                            <styled-content style="color:#214A88">color =</styled-content> 
                            <styled-content style="color:#6F9A04">"red"</styled-content>, 
                            <styled-content style="color:#214A88">linetype =</styled-content> 
                            <styled-content style="color:#6F9A04">"dashed"</styled-content>) 
                            <bold>
                                <styled-content style="color:#8F5A03">+</styled-content>
                            </bold>
                        </monospace>

                        <monospace>  
                            <bold>
                                <styled-content style="color:#214A88">theme_bw</styled-content>
                            </bold>()</monospace>
                    </preformat>
                </p>
                <fig fig-type="figure" id="f8" orientation="portrait" position="float">
                    <label>Figure 8. </label>
                    <caption>
                        <title>ssFC of potential key genes driving the inhibition of &#x201c;SUMOylation of DNA replication proteins&#x201d; pathway.</title>
                        <p>Single-sample logFC (ssFC) of potential key genes driving the inhibition of &#x201c;SUMOylation of DNA replication proteins&#x201d; pathway as a response to chemotherapy in HGSOC tumours.</p>
                    </caption>
                    <graphic id="gr8" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/163964/7d855428-3c1f-4fa3-a6c5-ae5e6351b94b_figure8.gif"/>
                </fig>
            </sec>
            <sec id="sec13">
                <title>Comparison to other pathway enrichment methods</title>
                <p>We also performed pathway analysis on this example dataset using three other methods to compare their performance against 
                    <italic toggle="yes">sSNAPPY.</italic> Details on the implementation of those methods and the comparisons performed are available at 
                    <ext-link ext-link-type="uri" xlink:href="https://zenodo.org/doi/10.5281/zenodo.10059951">10.5281/zenodo.10127829</ext-link>. We initially performed an analysis using SPIA.
                    <sup>
                        <xref ref-type="bibr" rid="ref6">6</xref>
                    </sup> However, SPIA relies on differentially expressed genes and given that only 49 genes were identified in our analysis using conventional differential expression analysis, no pathways were considered to be significant using SPIA. Additionally, we also performed pathway analysis on this example dataset using two non-topological-based approaches: 1) GSEA
                    <sup>
                        <xref ref-type="bibr" rid="ref3">3</xref>
                    </sup> using ranking statistics derived from differential expression analysis and 2) the fast version of rotation gene set testing for linear models (roast)
                    <sup>
                        <xref ref-type="bibr" rid="ref34">34</xref>
                    </sup> fry, neither of which rely on the presence of differentially expressed genes, Importantly, both methods test for signal within genes at either the up-regulated, or down-regulated extremes. Of the 219 gene sets considered associated with down-regulated genes by GSEA, 61 were considered as inhibited using 
                    <italic toggle="yes">sSNAPPY.</italic> Similarly, of the 21 pathways considered as associated with up-regulated genes by GSEA, 5 were considered as activated using 
                    <italic toggle="yes">sSNAPPY.</italic> GSEA produced a further 173 gene-sets not detected by 
                    <italic toggle="yes">sSNAPPY</italic>, whilst 
                    <italic toggle="yes">sSNAPPY</italic> produced an additional 54 pathways of interest.</p>
                <p>Analysis using fry yielded 117 pathways associated with down-regulated genes, with 
                    <italic toggle="yes">sSNAPPY</italic> considering 36 of these as inhibited. However, 5 pathways classified as inhibited under fry were considered as activated under 
                    <italic toggle="yes">sSNAPPY</italic>, highlighting that down-regulation of some genes may lead to activation of a pathway, which is vital information not available under fry. Similarly, for the 13 pathways associated with up-regulation under fry, two were considered activated by 
                    <italic toggle="yes">sSNAPPY</italic>, with one considered inhibited. A further 88 pathways were detected as being of interest under fry, without being considered as activated or inhibited by 
                    <italic toggle="yes">sSNAPPY</italic>, with 77 pathways uniquely detected under 
                    <italic toggle="yes">sSNAPPY.</italic>
                </p>
            </sec>
        </sec>
        <sec id="sec14" sec-type="discussion">
            <title>Discussion</title>
            <p>In conclusion, we have presented and provided a demonstration for the R/Bioconductor package 
                <italic toggle="yes">sSNAPPY</italic> which offers a novel single-sample pathway perturbation testing approach, tailored for heterogeneous tissue samples where experiments are performed using a matched-pair design. In contrast to many common enrichment methods, 
                <italic toggle="yes">sSNAPPY</italic> uses pathway topology information to compute perturbation scores which indicate the likely impact on the activity of a pathway, by predicting direction of change and enabling deeper characterisation of biological responses. By applying 
                <italic toggle="yes">sSNAPPY</italic> to a public scRNA-seq data collected before and after HGSOC patients were subjected to chemotherapy, we demonstrated its ability to detect significant pathway perturbations of various interesting biological processes consistent with, and far beyond what was shown in the original study. Whilst initially conceived for bulk-RNA studies, this demonstration has also provided clear applicability to scRNA datasets when using using pseudo-bulk approaches 
                <italic toggle="yes">sSNAPPY</italic> addresses the limitations of alternative strategies which fail to account for gene-gene interactions encoded by pathway topologies and are unable to predict the direction of pathway activities, nor the cumulative effect of expression change across multiple genes. In addition, the single-sample nature of the method can be utilised to address the increasing demand for personalised medicine. Through identifying shared and divergent responses between individuals, 
                <italic toggle="yes">sSNAPPY</italic> can provide valuable insights into the heterogeneous responses across clinical samples. Overall, we believe 
                <italic toggle="yes">sSNAPPY</italic> represents a valuable addition to the existing body of pathway analysis methods.</p>
        </sec>
        <sec id="sec15">
            <title>Ethics and consent</title>
            <p>Ethical approval and consent were not required.</p>
        </sec>
        <sec id="sec16">
            <title>Author contributions</title>
            <p>WL&#x2019;s contributions include Conceptualization, Data Curation, Formal Analysis, Investigation, Methodology, Project Administration, Software, Validation, Visualisation, Writing - Original Draft Preparation, and Writing - Review &amp; Editing. VM was involved with Conceptualization, Methodology and Writing - Review &amp; Editing. WDT contributed to Writing - Review &amp; Editing. SMP&#x2019;s contributions include Conceptualization, Methodology, Project Administration, Software, Supervision, Writing - Original Draft Preparation, and Writing - Review &amp; Editing.</p>
        </sec>
    </body>
    <back>
        <sec id="sec19" sec-type="data-availability">
            <title>Data availability</title>
            <p>The processed dataset used in this manuscript, along with the code used for data preparation is available at 
                <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.5281/zenodo.10867706">https://doi.org/10.5281/zenodo.10867706</ext-link>.</p>
            <p>Raw data was obtained from Gene Expression Omnibus: Longitudinal single-cell RNA-seq data of metastatic ovarian cancer. Accession GSE165897; 
                <ext-link ext-link-type="uri" xlink:href="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE165897">https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE165897</ext-link>.
                <sup>

                    <xref ref-type="bibr" rid="ref21">21</xref>
</sup>
            </p>
        </sec>
        <sec id="sec20">
            <title>Software availability</title>
            <p>

                <list list-type="bullet">
                    <list-item>
                        <label>&#x2022;</label>
                        <p>Software available from: 
                            <ext-link ext-link-type="uri" xlink:href="https://bioconductor.org/packages/release/bioc/html/sSNAPPY.html">https://bioconductor.org/packages/release/bioc/html/sSNAPPY.html</ext-link>
                        </p>
                    </list-item>
                    <list-item>
                        <label>&#x2022;</label>
                        <p>Source code available from: 
                            <ext-link ext-link-type="uri" xlink:href="https://github.com/Wenjun-Liu/sSNAPPY">https://github.com/Wenjun-Liu/sSNAPPY</ext-link>
                        </p>
                    </list-item>
                    <list-item>
                        <label>&#x2022;</label>
                        <p>Archived source code at time of publication: 
                            <ext-link ext-link-type="uri" xlink:href="https://zenodo.org/doi/10.5281/zenodo.8185450">https://zenodo.org/doi/10.5281/zenodo.8185450</ext-link>
                        </p>
                    </list-item>
                    <list-item>
                        <label>&#x2022;</label>
                        <p>License: GNU General Public License v3.0 
                            <ext-link ext-link-type="uri" xlink:href="https://opensource.org/license/gpl-3-0/">GPL-3</ext-link>
                        </p>
                    </list-item>
                </list>
            </p>
        </sec>
        <ref-list>
            <title>References</title>
            <ref id="ref1">
                <label>1</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Maleki</surname>
                            <given-names>F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ovens</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hogan</surname>
                            <given-names>DJ</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Gene set analysis: Challenges, opportunities, and future research.</article-title>
                    <source>
                        <italic toggle="yes">Front. Genet.</italic>
                    </source>
                    <year>2020</year>;<volume>11</volume>:<fpage>654</fpage>.
                    <pub-id pub-id-type="pmid">32695141</pub-id>
                    <pub-id pub-id-type="doi">10.3389/fgene.2020.00654</pub-id>
                    <pub-id pub-id-type="pmcid">PMC7339292</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref2">
                <label>2</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mubeen</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Tom Kodamullil</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hofmann-Apitius</surname>
                            <given-names>M</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>On the influence of several factors on pathway enrichment analysis.</article-title>
                    <source>
                        <italic toggle="yes">Brief. Bioinform.</italic>
                    </source>
                    <year>2022</year>;<volume>23</volume>(<issue>3</issue>):<fpage>bbac143</fpage>.
                    <pub-id pub-id-type="pmid">35453140</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bib/bbac143</pub-id>
                    <pub-id pub-id-type="pmcid">PMC9116215</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref3">
                <label>3</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Subramanian</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Tamayo</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Mootha</surname>
                            <given-names>VK</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Gene set enrichment analysis: A knowledge-based approach for interpreting genome-wide expression profiles.</article-title>
                    <source>
                        <italic toggle="yes">Proc. Natl. Acad. Sci. USA.</italic>
                    </source>
                    <year>2005 Oct</year>;<volume>102</volume>(<issue>43</issue>):<fpage>15545</fpage>&#x2013;<lpage>15550</lpage>.
                    <pub-id pub-id-type="pmid">16199517</pub-id>
                    <pub-id pub-id-type="doi">10.1073/pnas.0506580102</pub-id>
                    <pub-id pub-id-type="pmcid">PMC1239896</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref4">
                <label>4</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ogata</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Goto</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sato</surname>
                            <given-names>K</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>KEGG: Kyoto Encyclopedia of Genes and Genomes.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>1999 Jan</year>;<volume>27</volume>(<issue>1</issue>):<fpage>29</fpage>&#x2013;<lpage>34</lpage>.
                    <pub-id pub-id-type="pmid">9847135</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/27.1.29</pub-id>
                    <pub-id pub-id-type="pmcid">PMC148090</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref5">
                <label>5</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Martens</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ammar</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Riutta</surname>
                            <given-names>A</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>WikiPathways: Connecting communities.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2021 Jan</year>;<volume>49</volume>(<issue>D1</issue>):<fpage>D613</fpage>&#x2013;<lpage>D621</lpage>.
                    <pub-id pub-id-type="pmid">33211851</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gkaa1024</pub-id>
                    <pub-id pub-id-type="pmcid">PMC7779061</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref6">
                <label>6</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Tarca</surname>
                            <given-names>AL</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Draghici</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Khatri</surname>
                            <given-names>P</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>A novel signaling pathway impact analysis.</article-title>
                    <source>
                        <italic toggle="yes">Bioinformatics.</italic>
                    </source>
                    <year>2009 Jan</year>;<volume>25</volume>(<issue>1</issue>):<fpage>75</fpage>&#x2013;<lpage>82</lpage>.
                    <pub-id pub-id-type="pmid">18990722</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btn577</pub-id>
                    <pub-id pub-id-type="pmcid">PMC2732297</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref7">
                <label>7</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Jacob</surname>
                            <given-names>L</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Neuvial</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Dudoit</surname>
                            <given-names>S</given-names>
                        </name>
                    </person-group>:
                    <article-title>More power via graph-structured tests for differential expression of gene networks.</article-title>
                    <source>
                        <italic toggle="yes">Ann. Appl. Stat.</italic>
                    </source>
                    <year>2012 Jun</year>;<volume>6</volume>(<issue>2</issue>):<fpage>561</fpage>&#x2013;<lpage>600</lpage>.
                    <pub-id pub-id-type="doi">10.1214/11-AOAS528</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref8">
                <label>8</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ma</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shojaie</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Michailidis</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>:
                    <article-title>Network-based pathway enrichment analysis with incomplete network information.</article-title>
                    <source>
                        <italic toggle="yes">Bioinformatics.</italic>
                    </source>
                    <year>2016 Jun</year>;<volume>32</volume>(<issue>20</issue>):<fpage>3165</fpage>&#x2013;<lpage>3174</lpage>.
                    <pub-id pub-id-type="pmid">27357170</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btw410</pub-id>
                    <pub-id pub-id-type="pmcid">PMC5939912</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref9">
                <label>9</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ibrahim</surname>
                            <given-names>MAH</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jassim</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cawthorne</surname>
                            <given-names>MA</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>A topology-based score for pathway enrichment.</article-title>
                    <source>
                        <italic toggle="yes">J. Comput. Biol.</italic>
                    </source>
                    <year>2012 May</year>;<volume>19</volume>(<issue>5</issue>):<fpage>563</fpage>&#x2013;<lpage>573</lpage>.
                    <pub-id pub-id-type="pmid">22468678</pub-id>
                    <pub-id pub-id-type="doi">10.1089/cmb.2011.0182</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref10">
                <label>10</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Wu</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Smyth</surname>
                            <given-names>GK</given-names>
                        </name>
                    </person-group>:
                    <article-title>Camera: a competitive gene set test accounting for inter-gene correlation.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2012 May</year>;<volume>40</volume>(<issue>17</issue>):<fpage>e133</fpage>&#x2013;<lpage>e133</lpage>.
                    <pub-id pub-id-type="pmid">22638577</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gks461</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3458527</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref11">
                <label>11</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Nguyen</surname>
                            <given-names>TM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shafi</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Nguyen</surname>
                            <given-names>T</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Identifying significantly impacted pathways: A comprehensive review and assessment.</article-title>
                    <source>
                        <italic toggle="yes">Genome Biol.</italic>
                    </source>
                    <year>2019 Oct</year>;<volume>20</volume>(<issue>1</issue>):<fpage>203</fpage>.
                    <pub-id pub-id-type="pmid">31597578</pub-id>
                    <pub-id pub-id-type="doi">10.1186/s13059-019-1790-4</pub-id>
                    <pub-id pub-id-type="pmcid">PMC6784345</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref12">
                <label>12</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ma</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shojaie</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Michailidis</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>:
                    <article-title>A comparative study of topology-based pathway enrichment analysis methods.</article-title>
                    <source>
                        <italic toggle="yes">BMC Bioinformatics.</italic>
                    </source>
                    <year>2019 Dec</year>;<volume>20</volume>(<issue>1</issue>):<fpage>546</fpage>.
                    <pub-id pub-id-type="pmid">31684881</pub-id>
                    <pub-id pub-id-type="doi">10.1186/s12859-019-3146-1</pub-id>
                    <pub-id pub-id-type="pmcid">PMC6829999</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref13">
                <label>13</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>H&#x00e4;nzelmann</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Castelo</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Guinney</surname>
                            <given-names>J</given-names>
                        </name>
                    </person-group>:
                    <article-title>GSVA: Gene set variation analysis for microarray and RNA-Seq data.</article-title>
                    <source>
                        <italic toggle="yes">BMC Bioinformatics.</italic>
                    </source>
                    <year>2013 Dec</year>;<volume>14</volume>(<issue>1</issue>):<fpage>7</fpage>.
                    <pub-id pub-id-type="pmid">23323831</pub-id>
                    <pub-id pub-id-type="doi">10.1186/1471-2105-14-7</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3618321</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref14">
                <label>14</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Law</surname>
                            <given-names>CW</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Chen</surname>
                            <given-names>Y</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shi</surname>
                            <given-names>W</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Voom: Precision weights unlock linear model analysis tools for RNA-seq read counts.</article-title>
                    <source>
                        <italic toggle="yes">Genome Biol.</italic>
                    </source>
                    <year>2014</year>;<volume>15</volume>(<issue>2</issue>):<fpage>R29</fpage>.
                    <pub-id pub-id-type="pmid">24485249</pub-id>
                    <pub-id pub-id-type="doi">10.1186/gb-2014-15-2-r29</pub-id>
                    <pub-id pub-id-type="pmcid">PMC4053721</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref15">
                <label>15</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Knijnenburg</surname>
                            <given-names>TA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Wessels</surname>
                            <given-names>LFA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Reinders</surname>
                            <given-names>MJT</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Fewer permutations, more accurate P-values.</article-title>
                    <source>
                        <italic toggle="yes">Bioinformatics.</italic>
                    </source>
                    <year>2009 May</year>;<volume>25</volume>(<issue>12</issue>):<fpage>i161</fpage>&#x2013;<lpage>i168</lpage>.
                    <pub-id pub-id-type="pmid">19477983</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btp211</pub-id>
                    <pub-id pub-id-type="pmcid">PMC2687965</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref16">
                <label>16</label>
                <mixed-citation publication-type="other">
                    <collab>Gene set enrichment analysis (GSEA) User Guide</collab>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.gsea-msigdb.org/gsea/doc/GSEAUserGuideFrame.html">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref17">
                <label>17</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Smyth</surname>
                            <given-names>GK</given-names>
                        </name>
                    </person-group>:
                    <article-title>Linear models and empirical bayes methods for assessing differential expression in microarray experiments.</article-title>
                    <source>
                        <italic toggle="yes">Stat. Appl. Genet. Mol. Biol.</italic>
                    </source>
                    <year>2004 Feb</year>;<volume>3</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>25</lpage>.
                    <pub-id pub-id-type="pmid">16646809</pub-id>
                    <pub-id pub-id-type="doi">10.2202/1544-6115.1027</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref18">
                <label>18</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ritchie</surname>
                            <given-names>ME</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Phipson</surname>
                            <given-names>B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Wu</surname>
                            <given-names>D</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>limma powers differential expression analyses for RNA-sequencing and microarray studies.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2015 Apr</year>;<volume>43</volume>(<issue>7</issue>):<fpage>e47</fpage>.
                    <pub-id pub-id-type="pmid">25605792</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gkv007</pub-id>
                    <pub-id pub-id-type="pmcid">PMC4402510</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref19">
                <label>19</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Sales</surname>
                            <given-names>G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Calura</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cavalieri</surname>
                            <given-names>D</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Graphite - a Bioconductor package to convert pathway topology to gene network.</article-title>
                    <source>
                        <italic toggle="yes">BMC Bioinformatics.</italic>
                    </source>
                    <year>2012 Dec</year>;<volume>13</volume>(<issue>1</issue>):<fpage>20</fpage>.
                    <pub-id pub-id-type="pmid">22292714</pub-id>
                    <pub-id pub-id-type="doi">10.1186/1471-2105-13-20</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3296647</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref20">
                <label>20</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Gillespie</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jassal</surname>
                            <given-names>B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Stephan</surname>
                            <given-names>R</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>The reactome pathway knowledgebase 2022.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2021 Nov</year>;<volume>50</volume>(<issue>D1</issue>):<fpage>D687</fpage>&#x2013;<lpage>D692</lpage>.
                    <pub-id pub-id-type="pmid">34788843</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gkab1028</pub-id>
                    <pub-id pub-id-type="pmcid">PMC8689983</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref21">
                <label>21</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Zhang</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Erkan</surname>
                            <given-names>EP</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jamalzadeh</surname>
                            <given-names>S</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Longitudinal single-cell RNA-seq analysis reveals stress-promoted chemoresistance in metastatic ovarian cancer.</article-title>
                    <source>
                        <italic toggle="yes">Sci. Adv.</italic>
                    </source>
                    <year>2022 Feb</year>;<volume>8</volume>(<issue>8</issue>):<fpage>eabm1831</fpage>.
                    <pub-id pub-id-type="pmid">35196078</pub-id>
                    <pub-id pub-id-type="doi">10.1126/sciadv.abm1831</pub-id>
                    <pub-id pub-id-type="pmcid">PMC8865800</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref22">
                <label>22</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Hansen</surname>
                            <given-names>KD</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Irizarry</surname>
                            <given-names>RA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Wu</surname>
                            <given-names>Z</given-names>
                        </name>
                    </person-group>:
                    <article-title>Removing technical variability in RNA-seq data using conditional quantile normalization.</article-title>
                    <source>
                        <italic toggle="yes">Biostatistics.</italic>
                    </source>
                    <year>2012 Apr</year>;<volume>13</volume>(<issue>2</issue>):<fpage>204</fpage>&#x2013;<lpage>216</lpage>.
                    <pub-id pub-id-type="pmid">22285995</pub-id>
                    <pub-id pub-id-type="doi">10.1093/biostatistics/kxr054</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3297825</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref23">
                <label>23</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kamburov</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Stelzl</surname>
                            <given-names>U</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lehrach</surname>
                            <given-names>H</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>The ConsensusPathDB interaction database: 2013 update.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2012 Nov</year>;<volume>41</volume>(<issue>D1</issue>):<fpage>D793</fpage>&#x2013;<lpage>D800</lpage>.
                    <pub-id pub-id-type="pmid">23143270</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gks1055</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3531102</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref24">
                <label>24</label>
                <mixed-citation publication-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Wickham</surname>
                            <given-names>H</given-names>
                        </name>
                    </person-group>:
                    <source>
                        <italic toggle="yes">ggplot2: Elegant Graphics for Data Analysis.</italic>
                    </source>
                    <publisher-loc>New York, NY</publisher-loc>:
                    <publisher-name>Springer New York</publisher-name>;<year>2009</year>.
                    <pub-id pub-id-type="doi">10.1007/978-0-387-98141-3</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref25">
                <label>25</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Newman</surname>
                            <given-names>MEJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Girvan</surname>
                            <given-names>M</given-names>
                        </name>
                    </person-group>:
                    <article-title>Finding and evaluating community structure in networks.</article-title>
                    <source>
                        <italic toggle="yes">Phys. Rev. E.</italic>
                    </source>
                    <year>2004 Feb</year>;<volume>69</volume>(<issue>2</issue>):<fpage>026113</fpage>.
                    <pub-id pub-id-type="doi">10.1103/PhysRevE.69.026113</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref26">
                <label>26</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cunningham</surname>
                            <given-names>F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Allen</surname>
                            <given-names>JE</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Allen</surname>
                            <given-names>J</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Ensembl 2022.</article-title>
                    <source>
                        <italic toggle="yes">Nucleic Acids Res.</italic>
                    </source>
                    <year>2021 Nov</year>;<volume>50</volume>(<issue>D1</issue>):<fpage>D988</fpage>&#x2013;<lpage>D995</lpage>.
                    <pub-id pub-id-type="pmid">34791404</pub-id>
                    <pub-id pub-id-type="doi">10.1093/nar/gkab1049</pub-id>
                    <pub-id pub-id-type="pmcid">PMC8728283</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref27">
                <label>27</label>
                <mixed-citation publication-type="other">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kolde</surname>
                            <given-names>R</given-names>
                        </name>
                    </person-group>:
                    <article-title>Pheatmap: Pretty heatmaps.</article-title>
                    <year>2019</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://CRAN.R-project.org/package=pheatmap">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref28">
                <label>28</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Dong</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pang</surname>
                            <given-names>X</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Xu</surname>
                            <given-names>Y</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Ubiquitin-Conjugating Enzyme 9 Promotes Epithelial Ovarian Cancer Cell Proliferation in Vitro.</article-title>
                    <source>
                        <italic toggle="yes">Int. J. Mol. Sci.</italic>
                    </source>
                    <year>2013 May</year>;<volume>14</volume>(<issue>6</issue>):<fpage>11061</fpage>&#x2013;<lpage>11071</lpage>.
                    <pub-id pub-id-type="pmid">23708104</pub-id>
                    <pub-id pub-id-type="doi">10.3390/ijms140611061</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3709718</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref29">
                <label>29</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Qi</surname>
                            <given-names>G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Zhang</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ma</surname>
                            <given-names>H</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>CDCA8, targeted by MYBL2, promotes malignant progression and olaparib insensitivity in ovarian cancer.</article-title>
                    <source>
                        <italic toggle="yes">Am. J. Cancer Res.</italic>
                    </source>
                    <year>2021</year>;<volume>11</volume>(<issue>2</issue>):<fpage>389</fpage>&#x2013;<lpage>415</lpage>.
                    <pub-id pub-id-type="pmid">33575078</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref30">
                <label>30</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Zou</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Xu</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Li</surname>
                            <given-names>F</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Increased expression of UBE2T predicting poor survival of ovarian cancer: Based on bioinformatics analysis of UBE2s, clinical samples and the GEO database.</article-title>
                    <source>
                        <italic toggle="yes">DNA Cell Biol.</italic>
                    </source>
                    <year>2021</year>;<volume>40</volume>(<issue>1</issue>):<fpage>36</fpage>&#x2013;<lpage>60</lpage>.
                    <pub-id pub-id-type="pmid">33180631</pub-id>
                    <pub-id pub-id-type="doi">10.1089/dna.2020.5823</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref31">
                <label>31</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>G&#x0105;sowska-Bajger</surname>
                            <given-names>B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>G&#x0105;sowska-Bodnar</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Knapp</surname>
                            <given-names>P</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Prognostic Significance of Survivin Expression in Patients with Ovarian Carcinoma: A Meta-Analysis.</article-title>
                    <source>
                        <italic toggle="yes">J. Clin. Med.</italic>
                    </source>
                    <year>2021 Feb</year>;<volume>10</volume>(<issue>4</issue>):<fpage>879</fpage>.
                    <pub-id pub-id-type="pmid">33669912</pub-id>
                    <pub-id pub-id-type="doi">10.3390/jcm10040879</pub-id>
                    <pub-id pub-id-type="pmcid">PMC7924601</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref32">
                <label>32</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Gao</surname>
                            <given-names>Y</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Zhao</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ren</surname>
                            <given-names>M</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>TOP2A Promotes Tumorigenesis of High-grade Serous Ovarian Cancer by Regulating the TGF-&#x03b2;/Smad Pathway.</article-title>
                    <source>
                        <italic toggle="yes">J. Cancer.</italic>
                    </source>
                    <year>2020</year>;<volume>11</volume>(<issue>14</issue>):<fpage>4181</fpage>&#x2013;<lpage>4192</lpage>.
                    <pub-id pub-id-type="pmid">32368301</pub-id>
                    <pub-id pub-id-type="doi">10.7150/jca.42736</pub-id>
                    <pub-id pub-id-type="pmcid">PMC7196274</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref33">
                <label>33</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Chekerov</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Klaman</surname>
                            <given-names>I</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Zafrakas</surname>
                            <given-names>M</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>Altered Expression Pattern of Topoisomerase II, in Ovarian Tumor Epithelial and Stromal Cells after Platinum-Based Chemotherapy.</article-title>
                    <source>
                        <italic toggle="yes">Neoplasia.</italic>
                    </source>
                    <year>2006 Jan</year>;<volume>8</volume>(<issue>1</issue>):<fpage>38</fpage>&#x2013;<lpage>45</lpage>.
                    <pub-id pub-id-type="pmid">16533424</pub-id>
                    <pub-id pub-id-type="doi">10.1593/neo.05580</pub-id>
                    <pub-id pub-id-type="pmcid">PMC1584288</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref34">
                <label>34</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Wu</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lim</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Vaillant</surname>
                            <given-names>F</given-names>
                        </name>
                        <etal/>
                    </person-group>:
                    <article-title>ROAST: Rotation gene set tests for complex microarray experiments.</article-title>
                    <source>
                        <italic toggle="yes">Bioinformatics.</italic>
                    </source>
                    <year>2010</year>;<volume>26</volume>:<fpage>2176</fpage>&#x2013;<lpage>2182</lpage>.
                    <pub-id pub-id-type="pmid">20610611</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btq401</pub-id>
                    <pub-id pub-id-type="pmcid">PMC2922896</pub-id>
                </mixed-citation>
            </ref>
        </ref-list>
    </back>
    <sub-article article-type="reviewer-report" id="report297693">
        <front-stub>
            <article-id pub-id-type="doi">10.5256/f1000research.163964.r297693</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Moulos</surname>
                        <given-names>Panagiotis</given-names>
                    </name>
                    <xref ref-type="aff" rid="r297693a1">1</xref>
                    <role>Referee</role>
                </contrib>
                <aff id="r297693a1">
                    <label>1</label>Institute for Fundamental Biomedical Research, Biomedical Sciences Research Center &#x2018;Alexander Fleming&#x2019;, Vari, Greece</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>18</day>
                <month>7</month>
                <year>2024</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Moulos P</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport297693" related-article-type="peer-reviewed-article" xlink:href="10.12688/f1000research.149494.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>The authors present sSNAPPY, a Bioconductor package for the interesting research field of determining expression changes in whole pathways given the topology, instead of only identifying over-represented gene groups with multiple functionalities. They also extend the case by taking into account individual samples instead of summarized conditions, as done in previous work.</p>
            <p> </p>
            <p> Overall, the article is well-written with well-described mathematical formulas and notation. The concept is well-presented and documented. However, as individual-sample based analysis does not take into account replication, it would be beneficial if the authors would elaborate more on the issue, i.e. elaborate more on the theoretical foundations of their permutation strategy.</p>
            <p> </p>
            <p> Regarding the package itself, if accepted to Bioconductor there has already been a thorough review involving multiple code quality and consistency checks as well as testing, therefore there are no objections in this part. However, the authors should justify further their selection of a scRNA-Seq dataset instead of a bulk RNA-Seq one, as also stated in p. 6.</p>
            <p> </p>
            <p> Code related remarks:</p>
            <p> - To make the example more easily reproducible, code in the beginning of p. 7 should include downloading and renaming (if required) logCPM.tsv</p>
            <p> - retrieve_topology takes too much time. Maybe the authors could create a resource and submit as a different package to Bioconductor. Also, I got warnings while executing, the message is not very clear.</p>
            <p> - genePertScore, ssPertScore: numbers are not fully reproducible (Ubuntu 22.04, R 4.3.2), it should be made clear by the authors that the examples in the paper could be dependent on R version and other package versions</p>
            <p>Are the conclusions about the tool and its performance adequately supported by the findings presented in the article?</p>
            <p>Yes</p>
            <p>Is the rationale for developing the new software tool clearly explained?</p>
            <p>Yes</p>
            <p>Is the description of the software tool technically sound?</p>
            <p>Yes</p>
            <p>Are sufficient details of the code, methods and analysis (if applicable) provided to allow replication of the software development and its use by others?</p>
            <p>Yes</p>
            <p>Is sufficient information provided to allow interpretation of the expected output datasets and any results generated using the tool?</p>
            <p>Yes</p>
            <p>Reviewer Expertise:</p>
            <p>bioinformatics, clinical bioinformatics, transcriptomics, genetics</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report292642">
        <front-stub>
            <article-id pub-id-type="doi">10.5256/f1000research.163964.r292642</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Strbenac</surname>
                        <given-names>Dario</given-names>
                    </name>
                    <xref ref-type="aff" rid="r292642a1">1</xref>
                    <role>Referee</role>
                </contrib>
                <aff id="r292642a1">
                    <label>1</label>The University of Sydney, Sydney, New South Wales, Australia</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>4</day>
                <month>7</month>
                <year>2024</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Strbenac D</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport292642" related-article-type="peer-reviewed-article" xlink:href="10.12688/f1000research.149494.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>sSNAPPY in a Bioconductor package for directional pathway analysis. Having been accepted into Bioconductor, the package code and documentation is already known to be of good quality. Mathematical formulas are clear and the variables are all defined. However, the first step has an unfavourable user experience.</p>
            <p> </p>
            <p> I ran&#x00a0;retrieve_topology(database = "reactome", species = "hsapiens") but nothing happened for about half an hour and there were no progress nor error messages. I tried both a Windows 11 personal computer and a university-managed Linux server. Eventually, the function appears to complete successfully, albeit with a list of warnings of unknown consequence but seemingly ominous:</p>
            <p> </p>
            <p> &gt; head(warnings(), 3)</p>
            <p> Warning messages:</p>
            <p> 1: In FUN(X[[i]], ...) :</p>
            <p> &#x00a0; the conversion lost all edges of pathway "Uncoating of the HIV Virion"</p>
            <p> 2: In FUN(X[[i]], ...) :</p>
            <p> &#x00a0; the conversion lost all edges of pathway "Plus-strand DNA synthesis"</p>
            <p> 3: In FUN(X[[i]], ...) :</p>
            <p> &#x00a0; the conversion lost all edges of pathway "Virus Assembly and Release"</p>
            <p> </p>
            <p> If the user runs the command again, it takes a long time, instead of being cached to disk the first time and loaded near-instantly the second. This part of the analysis could benefit from BiocFileCache.</p>
            <p> </p>
            <p> Generation of Figure 3 involves a lot of ggplot2 coding for quality control plots which seems generally useful. Perhaps this should be a reusable function that package users could concisely write in their own R Markdown files. Similarly, Figure 4 involves four steps which could be encapsulated into a package function for better reusability.</p>
            <p> </p>
            <p> In terms of the Introduction, perhaps&#x00a0;
                <bold>Pengyi Yang et al.(2014 [ref - 1])</bold>&#x00a0;
                <ext-link ext-link-type="uri" xlink:href="https://academic.oup.com/bioinformatics/article/30/6/808/286146">Direction Pathway Analysis of Large-scale Proteomics Data Reveals Novel Features of the Insulin Action Pathway</ext-link>, 
                <italic>Bioinformatics</italic>&#x00a0;should be discussed, and limitations noted.</p>
            <p>Are the conclusions about the tool and its performance adequately supported by the findings presented in the article?</p>
            <p>Yes</p>
            <p>Is the rationale for developing the new software tool clearly explained?</p>
            <p>Partly</p>
            <p>Is the description of the software tool technically sound?</p>
            <p>Yes</p>
            <p>Are sufficient details of the code, methods and analysis (if applicable) provided to allow replication of the software development and its use by others?</p>
            <p>Yes</p>
            <p>Is sufficient information provided to allow interpretation of the expected output datasets and any results generated using the tool?</p>
            <p>Yes</p>
            <p>Reviewer Expertise:</p>
            <p>Statistical Bioinformatics</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
        <back>
            <ref-list>
                <title>References</title>
                <ref id="rep-ref-292642-1">
                    <label>1</label>
                    <mixed-citation publication-type="journal">
                        <person-group person-group-type="author"/>:
                        <article-title>Direction pathway analysis of large-scale proteomics data reveals novel features of the insulin action pathway.</article-title>
                        <source>
                            <italic>Bioinformatics</italic>
                        </source>.<year>2014</year>;<volume>30</volume>(<issue>6</issue>) :
                        <elocation-id>10.1093/bioinformatics/btt616</elocation-id>
                        <fpage>808</fpage>-<lpage>14</lpage>
                        <pub-id pub-id-type="pmid">24167158</pub-id>
                        <pub-id pub-id-type="doi">10.1093/bioinformatics/btt616</pub-id>
                    </mixed-citation>
                </ref>
            </ref-list>
        </back>
    </sub-article>
</article>
