<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.2 20190208//EN" "http://jats.nlm.nih.gov/publishing/1.2/JATS-journalpublishing1.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="other" dtd-version="1.2" xml:lang="en">
    <front>
        <journal-meta>
            <journal-id journal-id-type="pmc">F1000Research</journal-id>
            <journal-title-group>
                <journal-title>F1000Research</journal-title>
            </journal-title-group>
            <issn pub-type="epub">2046-1402</issn>
            <publisher>
                <publisher-name>F1000 Research Limited</publisher-name>
                <publisher-loc>London, UK</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="doi">10.12688/f1000research.52317.1</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Software Tool Article</subject>
                </subj-group>
                <subj-group>
                    <subject>Articles</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>PressPurt: network sensitivity to press perturbations under interaction uncertainty</article-title>
                <fn-group content-type="pub-status">
                    <fn>
                        <p>[version 1; peer review: awaiting peer review]</p>
                    </fn>
                </fn-group>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Koslicki</surname>
                        <given-names>David</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-0640-954X</uri>
                    <xref ref-type="corresp" rid="c1">a</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                    <xref ref-type="aff" rid="a2">2</xref>
                    <xref ref-type="aff" rid="a3">3</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Gibbon</surname>
                        <given-names>Dana</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a4">4</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Novak</surname>
                        <given-names>Mark</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-7881-4253</uri>
                    <xref ref-type="aff" rid="a5">5</xref>
                </contrib>
                <aff id="a1">
                    <label>1</label>Computer Science and Engineering, Pennsylvania State University, University Park, Pennsylvania, 16802, USA</aff>
                <aff id="a2">
                    <label>2</label>Biology, Pennsylvania State University, University Park, Pennsylvania, 16802, USA</aff>
                <aff id="a3">
                    <label>3</label>Huck Institutes of the Life Sciences, Pennsylvania State University, University Park, Pennsylvania, 16802, USA</aff>
                <aff id="a4">
                    <label>4</label>Center for Genome Research and Biocomputing, Oregon State University,, Corvallis, OR, 97330, USA</aff>
                <aff id="a5">
                    <label>5</label>Department of Integrative Biology, Oregon State University, Corvallis, OR, 97330, USA</aff>
            </contrib-group>
            <author-notes>
                <corresp id="c1">
                    <label>a</label>
                    <email xlink:href="mailto:dmk333@psu.edu">dmk333@psu.edu</email>
                </corresp>
                <fn fn-type="conflict">
                    <p>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>11</day>
                <month>2</month>
                <year>2022</year>
            </pub-date>
            <pub-date pub-type="collection">
                <year>2022</year>
            </pub-date>
            <volume>11</volume>
            <elocation-id>173</elocation-id>
            <history>
                <date date-type="accepted">
                    <day>11</day>
                    <month>11</month>
                    <year>2021</year>
                </date>
            </history>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2022 Koslicki D et al.</copyright-statement>
                <copyright-year>2022</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <self-uri content-type="pdf" xlink:href="https://f1000research.com/articles/11-173/pdf"/>
            <abstract>
                <p>While the use of networks to understand how complex systems respond to perturbations is pervasive across scientific disciplines, the uncertainty associated with estimates of pairwise interaction strengths (edge weights) remains rarely considered. Mischaracterizations of interaction strength can lead to qualitatively incorrect predictions regarding system responses as perturbations propagate through often counteracting direct and indirect effects.</p>
                <p>Here, we introduce 
                    <monospace>PressPurt</monospace>, a computational package for identifying the interactions whose strengths must be estimated most accurately in order to produce robust predictions of a network's response to press perturbations. The package provides methods for calculating and visualizing these edge-specific sensitivities (tolerances) when uncertainty is associated to one or more edges according to a variety of different error distributions. The software requires the network to be represented as a numerical (quantitative or qualitative) Jacobian matrix evaluated at stable equilibrium.</p>
                <p>
                    <monospace>PressPurt</monospace> is open source under the MIT license and is available as both a Python package and an R package hosted at 
                    <ext-link ext-link-type="uri" xlink:href="https://github.com/dkoslicki/PressPurt">https://github.com/dkoslicki/PressPurt</ext-link> and on the CRAN repository 
                    <ext-link ext-link-type="uri" xlink:href="https://CRAN.R-project.org/package=PressPurt">https://CRAN.R-project.org/package=PressPurt</ext-link>.</p>
            </abstract>
            <kwd-group kwd-group-type="author">
                <kwd>Press perturbation</kwd>
                <kwd>sensitivity</kwd>
                <kwd>loop analysis</kwd>
                <kwd>uncertainty quantification</kwd>
            </kwd-group>
            <funding-group>
                <award-group id="fund-1" xlink:href="http://dx.doi.org/10.13039/100000001">
                    <funding-source>National Science Foundation</funding-source>
                    <award-id>1664803</award-id>
                </award-group>
                <funding-statement>This material is based upon work supported by the National Science Foundation under Grant No. 1664803 (PI David Koslicki).</funding-statement>
                <funding-statement>
                    <italic>The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.</italic>
                </funding-statement>
            </funding-group>
        </article-meta>
    </front>
    <body>
        <sec id="sec1" sec-type="intro">
            <title>Introduction</title>
            <p>Networks have become a routine tool for representing the complex systems that pervade biology, technology and society. While the development of methods for inferring network topology remains a dominant focus,
                <sup>
                    <xref ref-type="bibr" rid="ref18">18</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref19">19</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref21">21</xref>
                </sup>
                <sup>-</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref24">24</xref>
                </sup> numerous methods are now also being advanced for quantifying the weights (fluxes and interaction strengths) of the edges between pairs of nodes (genes, metabolites, species) in different network types.
                <sup>
                    <xref ref-type="bibr" rid="ref1">1</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref5">5</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref7">7</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref9">9</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref10">10</xref>
                </sup> Motivating advances is the desire to predict how systems respond to perturbations wherein the concentration or process rates of a subset of nodes is chronically altered by external forces.
                <sup>
                    <xref ref-type="bibr" rid="ref2">2</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref28">28</xref>
                </sup> Examples of such perturbations include gene knock-downs, fisheries harvest, and the administering of antibiotics.
                <sup>
                    <xref ref-type="bibr" rid="ref17">17</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref20">20</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref25">25</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref27">27</xref>
                </sup>
            </p>
            <p>Predictive insight into perturbation effects nevertheless remains hard to obtain. This is true even when a network&#x2019;s topology is fixed and precisely specified because edge weights are always subject to estimation error and empirical variation. Because of the many, often counteracting pathways through which perturbations travel, these uncertainties rapidly compound.
                <sup>
                    <xref ref-type="bibr" rid="ref16">16</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref26">26</xref>
                </sup> In many cases, therefore, predicting even just the qualitative sign pattern by which nodes respond to perturbations (i.e. whether concentrations increase, decrease, or remain unchanged) is of desirable interest.
                <sup>
                    <xref ref-type="bibr" rid="ref6">6</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref14">14</xref>
                </sup>
                <sup>,</sup>
                <sup>
                    <xref ref-type="bibr" rid="ref17">17</xref>
                </sup>
            </p>
            <p>Here we introduce 
                <monospace>PressPurt</monospace> &#x2013; a collection of computational tools designed to shed light on the qualitative and quantitative response of a network to press perturbations when there is uncertainty in the magnitude of edge weights. The 
                <monospace>PressPurt</monospace> package is written to implement in a usable form the theoretical results established in Ref. 
                <xref ref-type="bibr" rid="ref11">11</xref>. 
                <monospace>PressPurt</monospace> is designed to identify the most sensitive interactions within the network which must be estimated most accurately to produce robust predictions of press perturbation responses.</p>
        </sec>
        <sec id="sec2" sec-type="methods">
            <title>Methods</title>
            <sec id="sec3">
                <title>Scope</title>
                <p>
                    <monospace>PressPurt</monospace> is designed to analyze networks whose dynamics may be described by a system of differential equations of the form 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mfrac>
                                <mml:msub>
                                    <mml:mi mathvariant="italic">dN</mml:mi>
                                    <mml:mi>i</mml:mi>
                                </mml:msub>
                                <mml:mi mathvariant="italic">dt</mml:mi>
                            </mml:mfrac>
                            <mml:mo>=</mml:mo>
                            <mml:msub>
                                <mml:mi>f</mml:mi>
                                <mml:mi>i</mml:mi>
                            </mml:msub>
                            <mml:mfenced close=")" open="(">
                                <mml:mover accent="true">
                                    <mml:mi>N</mml:mi>
                                    <mml:mo stretchy="true">&#x2192;</mml:mo>
                                </mml:mover>
                            </mml:mfenced>
                            <mml:mo>+</mml:mo>
                            <mml:msub>
                                <mml:mi>u</mml:mi>
                                <mml:mi>i</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> for 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                            <mml:mo>=</mml:mo>
                            <mml:mn>1</mml:mn>
                            <mml:mo>,</mml:mo>
                            <mml:mo>&#x2026;</mml:mo>
                            <mml:mo>,</mml:mo>
                            <mml:mi>n</mml:mi>
                        </mml:math>
                    </inline-formula> variables. Here, 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>f</mml:mi>
                                <mml:mi>i</mml:mi>
                            </mml:msub>
                            <mml:mfenced close=")" open="(">
                                <mml:mover accent="true">
                                    <mml:mi>N</mml:mi>
                                    <mml:mo stretchy="true">&#x2192;</mml:mo>
                                </mml:mover>
                            </mml:mfenced>
                        </mml:math>
                    </inline-formula> is a function describing the interactions between node 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula> and a vector of other nodes, 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>N</mml:mi>
                                <mml:mi>i</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> is node 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula>&#x2019;s concentration, and 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mi>u</mml:mi>
                                <mml:mi>i</mml:mi>
                            </mml:msub>
                        </mml:math>
                    </inline-formula> is a scalar representing a constant rate of external input to (or removal from) node 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula> (a so-called 
                    <italic toggle="yes">press perturbation</italic>, sensu Ref. 
                    <xref ref-type="bibr" rid="ref2">2</xref>). Other forms of press perturbation may be represented in a similar manner.
                    <sup>
                        <xref ref-type="bibr" rid="ref17">17</xref>
                    </sup>
                </p>
                <p>For input, 
                    <monospace>PressPurt</monospace> requires only a single CSV file representing the so-called Community Matrix
                    <sup>
                        <xref ref-type="bibr" rid="ref12">12</xref>
                    </sup>: the Jacobian of the system evaluated at a stable equilibrium point, 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msub>
                                <mml:mfenced close="|" open="|">
                                    <mml:mrow>
                                        <mml:msub>
                                            <mml:mi>A</mml:mi>
                                            <mml:mrow>
                                                <mml:mi>i</mml:mi>
                                                <mml:mo>,</mml:mo>
                                                <mml:mi>j</mml:mi>
                                            </mml:mrow>
                                        </mml:msub>
                                        <mml:mo>=</mml:mo>
                                        <mml:mfrac>
                                            <mml:mrow>
                                                <mml:mi>&#x2202;</mml:mi>
                                                <mml:msub>
                                                    <mml:mi>f</mml:mi>
                                                    <mml:mi>i</mml:mi>
                                                </mml:msub>
                                                <mml:mfenced close=")" open="(">
                                                    <mml:mover accent="true">
                                                        <mml:mi>N</mml:mi>
                                                        <mml:mo stretchy="true">&#x2192;</mml:mo>
                                                    </mml:mover>
                                                </mml:mfenced>
                                            </mml:mrow>
                                            <mml:mrow>
                                                <mml:mi>&#x2202;</mml:mi>
                                                <mml:msub>
                                                    <mml:mi>N</mml:mi>
                                                    <mml:mi>j</mml:mi>
                                                </mml:msub>
                                            </mml:mrow>
                                        </mml:mfrac>
                                    </mml:mrow>
                                </mml:mfenced>
                                <mml:msup>
                                    <mml:mi>x</mml:mi>
                                    <mml:mo>&#x2217;</mml:mo>
                                </mml:msup>
                            </mml:msub>
                        </mml:math>
                    </inline-formula>. 
                    <monospace>PressPurt</monospace> begins by determining how a press perturbation on each node 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>j</mml:mi>
                        </mml:math>
                    </inline-formula> is predicted to alter the equilibrium concentration of each node 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>i</mml:mi>
                        </mml:math>
                    </inline-formula> by calculating the Net Effects matrix without interaction strength uncertainty, 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mfrac>
                                <mml:mrow>
                                    <mml:mi>&#x2202;</mml:mi>
                                    <mml:msub>
                                        <mml:mi>N</mml:mi>
                                        <mml:mi>i</mml:mi>
                                    </mml:msub>
                                </mml:mrow>
                                <mml:mrow>
                                    <mml:mi>&#x2202;</mml:mi>
                                    <mml:msub>
                                        <mml:mi>u</mml:mi>
                                        <mml:mi>j</mml:mi>
                                    </mml:msub>
                                </mml:mrow>
                            </mml:mfrac>
                            <mml:mo>=</mml:mo>
                            <mml:mo>&#x2212;</mml:mo>
                            <mml:msubsup>
                                <mml:mi>A</mml:mi>
                                <mml:mi mathvariant="italic">ij</mml:mi>
                                <mml:mrow>
                                    <mml:mo>&#x2212;</mml:mo>
                                    <mml:mn>1</mml:mn>
                                </mml:mrow>
                            </mml:msubsup>
                        </mml:math>
                    </inline-formula>
                    <sub>.</sub>
                    <sup>
                        <xref ref-type="bibr" rid="ref6">6</xref>
                    </sup>
                    <sup>,</sup>
                    <sup>
                        <xref ref-type="bibr" rid="ref17">17</xref>
                    </sup>
                    <sup>,</sup>
                    <sup>
                        <xref ref-type="bibr" rid="ref26">26</xref>
                    </sup> Uncertainties in the elements of 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mi>A</mml:mi>
                        </mml:math>
                    </inline-formula> that alter the sign structure of 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mo>&#x2212;</mml:mo>
                            <mml:msup>
                                <mml:mi>A</mml:mi>
                                <mml:mrow>
                                    <mml:mo>&#x2212;</mml:mo>
                                    <mml:mn>1</mml:mn>
                                </mml:mrow>
                            </mml:msup>
                        </mml:math>
                    </inline-formula> are the cause of qualitative mispredictions.
                    <sup>
                        <xref ref-type="bibr" rid="ref26">26</xref>
                    </sup>
                </p>
            </sec>
            <sec id="sec4">
                <title>Network sensitivity with single edge uncertainty</title>
                <p>
                    <monospace>PressPurt</monospace> can quantify the effects of interaction strength uncertainty in a single edge, independent of uncertainty in all other edges. To do so, the user specifies the kind of distribution and parameterization that describes interaction strength uncertainties. Currently, options include the uniform, truncated normal, truncated log normal, and beta distributions. Within the specified distribution, 
                    <monospace>PressPurt</monospace> finds the range of uncertainty values that permits the system to remain asymptotically stable, then returns a matrix in CSV format whose 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msup>
                                <mml:mfenced close=")" open="(" separators=",">
                                    <mml:mi>k</mml:mi>
                                    <mml:mi>l</mml:mi>
                                </mml:mfenced>
                                <mml:mi>th</mml:mi>
                            </mml:msup>
                        </mml:math>
                    </inline-formula> entry gives the expected (average) fraction of qualitative/sign changes in the net effects matrix resulting from the interaction uncertainty associated with edge 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:mfenced close=")" open="(" separators=",">
                                <mml:mi>k</mml:mi>
                                <mml:mi>l</mml:mi>
                            </mml:mfenced>
                        </mml:math>
                    </inline-formula>. 
                    <monospace>PressPurt</monospace> can produce a visualization of the exact number of sign switches that are incurred as a function of uncertainty magnitude for a given edge (
                    <xref ref-type="fig" rid="f1">Figure 1b</xref>), as well as a heat map visualization of all edges depicting the fraction of possible sign switches that are incurred by uncertainty in each edge (
                    <xref ref-type="fig" rid="f1">Figure 1c</xref>). The overall sensitivity of the network when interaction uncertainty is considered one edge at a time is also returned as a percent of all 
                    <inline-formula>
                        <mml:math display="inline">
                            <mml:msup>
                                <mml:mi>n</mml:mi>
                                <mml:mn>2</mml:mn>
                            </mml:msup>
                        </mml:math>
                    </inline-formula> possible mispredictions.</p>
                <fig fig-type="figure" id="f1" orientation="portrait" position="float">
                    <label>Figure 1. </label>
                    <caption>
                        <title>We demonstrate the 
                            <monospace>PressPurt</monospace> package on the 
                            <italic toggle="yes">Escherichia coli</italic> metabolic network inferred by Ref. 
                            <xref ref-type="bibr" rid="ref3">3</xref> and quantified in Jacobian form by Ref. 
                            <xref ref-type="bibr" rid="ref10">10</xref>.</title>
                        <p>Panel a) shows the network with positive and negative interactions depicted with arrowheads and balls respectively. Panel b) shows how uncertainty in the magnitude of the (pep, fdp) interaction (assuming a truncated normal uncertainty distribution, shown in gray) affects the number of qualitative mispredictions (sign changes in the net effects matrix, shown in blue) that are made regarding how the concentrations of the network&#x2019;s metabolites will respond to press perturbations. Panel c) shows a heat map of the expected (average) fraction of mispredictions that are incurred by uncertainty in each (
                            <italic toggle="yes">k</italic>, 
                            <italic toggle="yes">l</italic>) edge (i.e. for the (pep, fdp) edge, the integration of the blue line with respect to the gray distribution in panel b). It demonstrates that uncertainty in the direct effect of fructose-1,6-bisphosphate (fdp) on phosphoenolpyruvate (pep) will result, on average, in the most mispredictions in the net effects of a unit perturbation to any of the metabolites in the network).</p>
                    </caption>
                    <graphic id="gr1" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/55580/dfc12ab3-0104-48e8-9fde-1ad41466b6ab_figure1.gif"/>
                </fig>
                <p>Similar computation and visualization functions are provided to measure the quantitative response of the network when interaction strength uncertainties are at their stability-limited values. In this context the actual magnitude change in the net effects matrix is computed, as opposed to just quantifying sign-changes in the net effects matrix.</p>
            </sec>
            <sec id="sec5">
                <title>Network sensitivity with multiple edge uncertainty</title>
                <p>
                    <monospace>PressPurt</monospace> can also quantify the effects of interaction strength uncertainty when it is associated with multiple edges simultaneously. This is achieved via Monte Carlo sampling where uncertainties are presumed to follow a uniform distribution of a user-specified length. The fraction of samples that contain a qualitative misprediction in the net effects matrix is then reported.</p>
            </sec>
            <sec id="sec6">
                <title>Implementation</title>
                <p>
                    <monospace>PressPurt</monospace> is available as both a Python package and an R package hosted at 
                    <ext-link ext-link-type="uri" xlink:href="https://github.com/dkoslicki/PressPurt">
                        <monospace>https://github.com/dkoslicki/PressPurt</monospace>
                    </ext-link> and on the CRAN repository 
                    <ext-link ext-link-type="uri" xlink:href="https://CRAN.R-project.org/package=PressPurt">
                        <monospace>https://CRAN.R-project.org/package=PressPurt</monospace>
                    </ext-link>. These provide complete documentation of all functions, as well as detailed installation instructions, quick start and use tutorials with example data (i.e. Jacobian matrices).</p>
            </sec>
            <sec id="sec7">
                <title>Operation</title>
                <p>The 
                    <monospace>PressPurt</monospace> Python package is dependent on the Python packages 
                    <italic toggle="yes">numpy, scipy, matplotlib, sympy, pathos</italic> and 
                    <italic toggle="yes">pandas.</italic> The 
                    <monospace>PressPurt</monospace> R package is dependent on Python for its symbolic toolbox and uses the R package 
                    <italic toggle="yes">reticulate</italic> to communicate with Python. Thus Python, as well as its dependent packages, must be installed. It also depends on the R packages 
                    <italic toggle="yes">data.table, ggplot2, grid, gridExtra</italic> and 
                    <italic toggle="yes">utils</italic>, primarily for the convenience of their efficient data manipulation and visualization functions.</p>
            </sec>
        </sec>
        <sec id="sec8">
            <title>Use case</title>
            <p>For input, 
                <monospace>PressPurt</monospace> requires only a single CSV file representing either a quantitative or qualitatively-specified Jacobian matrix and the specification of a desired error distribution for the edge weight uncertainties. The three steps of an analysis entail: (1) preprocessing the Jacobian matrix, (2) computing the entry-wise or multi-entry perturbation expectations for either qualitative or quantitative sensitivities, and (3) visualizing the results. In R, for example, determining and visualizing the qualitative entry-wise sensitivities of a four-species intraguild predation module (whose Jacobian matrix
                <sup>
                    <xref ref-type="bibr" rid="ref11">11</xref>
                </sup>
                <disp-formula id="e1">
                    <mml:math display="block">
                        <mml:mfenced close="]" open="[">
                            <mml:mtable columnalign="center">
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>0.237</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>1.000</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.000</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.000</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.100</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>0.015</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>1.000</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>1.000</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.000</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.100</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>0.015</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>1.000</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.000</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.045</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.100</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mo>&#x2212;</mml:mo>
                                        <mml:mn>0.015</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                            </mml:mtable>
                        </mml:mfenced>
                    </mml:math>
                </disp-formula>is provided in 
                <monospace>PressPurt</monospace> as IGP.csv) involves three commands. First,</p>
            <p>
                <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
infile &lt;- system.file("extdata", "Modules", "IGP.csv",
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;package = "PressPurt")
PreProsMatrix &lt;- PreprocessMatrix (input_file = infile
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;max_bound = 10,
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;zero_perturb = FALSE,
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;threads = 2)</preformat>
            </p>
            <p>which returns information that includes, for each non-zero edge, their asymptotic stability intervals and values that would lead to a sign switch in the net effects matrix. Second,</p>
            <p>
                <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
Entrywise &lt;- ComputeEntryWisePerturbationExpectation (
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;PreProsMatrix = PreProsMatrix,
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;distribution_type = "truncnorm",
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;input_a = 0, input_b = -2, threads = 1)</preformat>
            </p>
            <p>which specifies a truncated normal distribution of edge weight uncertainties with mean 0 and variance 2 (a negative value indicates that the standard deviation is to be the scaled to each edge weight) and whose output includes a matrix containing the consequent expected number of sign switches in the net effects matrix expressed as a percentage of all 16 possible mis-predictions,
                <disp-formula id="e2">
                    <mml:math display="block">
                        <mml:mfenced close="]" open="[">
                            <mml:mtable columnalign="center">
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.03</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.06</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.00</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.00</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.06</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.12</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.05</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.02</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.00</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.03</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.16</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.01</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                                <mml:mtr>
                                    <mml:mtd>
                                        <mml:mn>0.00</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.05</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.08</mml:mn>
                                    </mml:mtd>
                                    <mml:mtd>
                                        <mml:mn>0.17</mml:mn>
                                    </mml:mtd>
                                </mml:mtr>
                            </mml:mtable>
                        </mml:mfenced>
                        <mml:mo>.</mml:mo>
                    </mml:math>
                </disp-formula>
            </p>
            <p>Third,</p>
            <p>
                <preformat orientation="portrait" position="float" preformat-type="computer code" xml:space="preserve">
GenerateEntryWiseFigures (EntryWise = Entrywise,
&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2003;&#x2002;&#x2003;all_numswitch_plots = TRUE)</preformat>
            </p>
            <p>which produces 
                <xref ref-type="fig" rid="f2">Figure 2</xref> that overlays visualizations of the assumed uncertainty distribution associated with each edge weight and the consequent number of qualitative mispredictions (sign switches) that a given magnitude of edge weight uncertainty will incur in the net effects matrix as a whole.</p>
            <fig fig-type="figure" id="f2" orientation="portrait" position="float">
                <label>Figure 2. </label>
                <caption>
                    <title>Visualization produced by the R package function 
                        <monospace>GenerateEntryWiseFigures</monospace> depicting the assumed uncertainty distribution associated with each non-zero Jacobian element in a four-species intraguild predation network (grey distributions) and the consequent number of qualitative mispredictions (sign switches) that a given magnitude of uncertainty in each element will incur in the net effects matrix of the network as a whole (black lines).</title>
                </caption>
                <graphic id="gr2" orientation="portrait" position="float" xlink:href="https://f1000research-files.f1000.com/manuscripts/55580/dfc12ab3-0104-48e8-9fde-1ad41466b6ab_figure2.gif"/>
            </fig>
        </sec>
        <sec id="sec9">
            <title>Summary</title>
            <p>The measures of sensitivity that 
                <monospace>PressPurt</monospace> implements are exact and may be computed with relative ease and computational efficiency. 
                <monospace>PressPurt</monospace> thereby obviates the need for what are typically computationally expensive simulations whose results can be difficult to interpret when assessing the sources of mispredictions in complex networks. Underlying these advances is the separation of uncertainty magnitudes from their frequency distributions (
                <xref ref-type="fig" rid="f1">Figure 1b</xref>). Moreover, the theorems of Ref. 
                <xref ref-type="bibr" rid="ref11">11</xref> show that the analysis of qualitative models via Loop Analysis
                <sup>
                    <xref ref-type="bibr" rid="ref13">13</xref>
                </sup> &#x2013; the most commonly applied tool in context of fisheries management and conservation science wherein the values of 
                <inline-formula>
                    <mml:math display="inline">
                        <mml:mi>A</mml:mi>
                    </mml:math>
                </inline-formula> are specified as either &#x2212;1, 0, or 1 &#x2013; are a special case of the methods implemented in 
                <monospace>PressPurt</monospace>. In this regard, 
                <monospace>PressPurt</monospace> complements the functionality of other network-based packages such as Refs. 
                <xref ref-type="bibr" rid="ref4">4</xref>, 
                <xref ref-type="bibr" rid="ref8">8</xref>, 
                <xref ref-type="bibr" rid="ref15">15</xref>.</p>
            <p>Future extensions of 
                <monospace>PressPurt</monospace> may include reformulating the code to: (1) compute which predictions in the net effects matrix are most sensitive to errors in focal (or multiple) entries of the Jacobian, (2) determine which interactions are most sensitive with respect to a specific net effects prediction, and (3) allow for different kinds of distributions or parameterizations to be specified for each interaction strength uncertainty in both the single and multiple edge computations. Lastly, other theorems contained in Ref. 
                <xref ref-type="bibr" rid="ref11">11</xref>, including those relating to the characterization of quantitative mispredictions, may also be implemented.</p>
        </sec>
        <sec id="sec10">
            <title>Data availability</title>
            <p>All data underlying the results are available as part of the article and no additional source data are required.</p>
        </sec>
        <sec id="sec11">
            <title>Software availability</title>
            <p>
                <monospace>PressPurt</monospace> is available from: 
                <ext-link ext-link-type="uri" xlink:href="https://cran.r-project.org/web/packages/PressPurt/">
                    <monospace>https://cran.r-project.org/web/packages/PressPurt/</monospace>
                </ext-link>
            </p>
            <p>Source code available from: 
                <ext-link ext-link-type="uri" xlink:href="https://github.com/dkoslicki/PressPurt">
                    <monospace>https://github.com/dkoslicki/PressPurt</monospace>
                </ext-link>
            </p>
            <p>Archived source code as at time of publication: 
                <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.5281/zenodo.5661173">
                    <monospace>https://doi.org/10.5281/zenodo.5661173</monospace>
                </ext-link>
                <sup>
                    <xref ref-type="bibr" rid="ref29">29</xref>
                </sup>
            </p>
            <p>License: MIT</p>
        </sec>
    </body>
    <back>
        <ref-list>
            <title>References</title>
            <ref id="ref1">
                <label>1</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Antoniewicz</surname>
                            <given-names>MR</given-names>
                        </name>
</person-group>:
                    <article-title>Methods and advances in metabolic flux analysis: a mini-review.</article-title>
                    <source>

                        <italic toggle="yes">J. Ind. Microbiol. Biotechnol.</italic>
</source>
                    <year>2015</year>;<volume>42</volume>(<issue>3</issue>):<fpage>317</fpage>&#x2013;<lpage>325</lpage>.
                    <pub-id pub-id-type="pmid">25613286</pub-id>
                    <pub-id pub-id-type="doi">10.1007/s10295-015-1585-x</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref2">
                <label>2</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bender</surname>
                            <given-names>EA</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Case</surname>
                            <given-names>TJ</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Gilpin</surname>
                            <given-names>ME</given-names>
                        </name>
</person-group>:
                    <article-title>Perturbation experiments in community ecology: theory and practice.</article-title>
                    <source>

                        <italic toggle="yes">Ecology.</italic>
</source>
                    <year>1984</year>;<volume>65</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>13</lpage>.
                    <pub-id pub-id-type="doi">10.2307/1939452</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref3">
                <label>3</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Chassagnole</surname>
                            <given-names>C</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Noisommit-Rizzi</surname>
                            <given-names>N</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Schmid</surname>
                            <given-names>JW</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Dynamic modeling of the central carbon metabolism of Escherichia coli.</article-title>
                    <source>

                        <italic toggle="yes">Biotechnol. Bioeng.</italic>
</source>
                    <year>2002</year>;<volume>79</volume>(<issue>1</issue>):<fpage>53</fpage>&#x2013;<lpage>73</lpage>.
                    <pub-id pub-id-type="pmid">17590932</pub-id>
                    <pub-id pub-id-type="doi">10.1002/bit.10288</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref4">
                <label>4</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Csardi</surname>
                            <given-names>G</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Nepusz</surname>
                            <given-names>T</given-names>
                        </name>
</person-group>:
                    <article-title>The igraph software package for complex network research.</article-title>
                    <source>

                        <italic toggle="yes">InterJournal, Complex Systems: 1695.</italic>
</source>
                    <year>2006</year>.</mixed-citation>
            </ref>
            <ref id="ref5">
                <label>5</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Dahlquist</surname>
                            <given-names>KD</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Fitzpatrick</surname>
                            <given-names>BG</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Camacho</surname>
                            <given-names>ET</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Parameter estimation for gene regulatory networks from microarray data: Cold shock response in 
                        <italic toggle="yes">Saccharomyces cerevisiae.</italic>
                    </article-title>
                    <source>

                        <italic toggle="yes">Bull. Math. Biol.</italic>
</source>
                    <year>2015</year>;<volume>77</volume>(<issue>8</issue>):<fpage>1457</fpage>&#x2013;<lpage>1492</lpage>.
                    <pub-id pub-id-type="pmid">26420504</pub-id>
                    <pub-id pub-id-type="doi">10.1007/s11538-015-0092-6</pub-id>
                    <pub-id pub-id-type="pmcid">PMC4636536</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref6">
                <label>6</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Dambacher</surname>
                            <given-names>JM</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Li</surname>
                            <given-names>HW</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Rossignol</surname>
                            <given-names>PA</given-names>
                        </name>
</person-group>:
                    <article-title>Relevance of community structure in assessing indeterminacy of ecological predictions.</article-title>
                    <source>

                        <italic toggle="yes">Ecology.</italic>
</source>
                    <year>2002</year>;<volume>83</volume>(<issue>5</issue>):<fpage>1372</fpage>&#x2013;<lpage>1385</lpage>.
                    <pub-id pub-id-type="doi">10.1890/0012-9658(2002)083[1372:ROCSIA]2.0.CO;2</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref7">
                <label>7</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Deyle</surname>
                            <given-names>ER</given-names>
                        </name>

                        <name name-style="western">
                            <surname>May Robert</surname>
                            <given-names>M</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Munch Stephan</surname>
                            <given-names>B</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Tracking and forecasting ecosystem interactions in real time.</article-title>
                    <source>

                        <italic toggle="yes">Proc. R. Soc. B Biol. Sci.</italic>
</source>
                    <year>2016</year>;<volume>283</volume>(<issue>1822</issue>)
                    <comment>20152258, 2019/06/12</comment>:<fpage>20152258</fpage>.
                    <pub-id pub-id-type="doi">10.1098/rspb.2015.2258</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref8">
                <label>8</label>
                <mixed-citation publication-type="other">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Dinno</surname>
                            <given-names>A</given-names>
                        </name>
</person-group>:
                    <article-title>Loopanalyst: A collection of tools to conduct levins&#x2019; loop analysis.</article-title>
                    <year>2018</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://cran.r-project.org/package=loopanalyst">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref9">
                <label>9</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Gauzens</surname>
                            <given-names>B</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Barnes</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Giling</surname>
                            <given-names>DP</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>fluxweb: An r package to easily estimate energy fluxes in food webs.</article-title>
                    <source>

                        <italic toggle="yes">Methods Ecol. Evol.</italic>
</source>
                    <year>2019/06/12 2019</year>;<volume>10</volume>(<issue>2</issue>):<fpage>270</fpage>&#x2013;<lpage>279</lpage>.
                    <pub-id pub-id-type="doi">10.1111/2041-210X.13109</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref10">
                <label>10</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Khatibipour</surname>
                            <given-names>MJ</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kurto&#x011f;lu</surname>
                            <given-names>F</given-names>
                        </name>

                        <name name-style="western">
                            <surname>&#x00c7;akir</surname>
                            <given-names>T</given-names>
                        </name>
</person-group>:
                    <article-title>Jacly: a jacobian-based method for the inference of metabolic interactions from the covariance of steady-state metabolome data.</article-title>
                    <source>

                        <italic toggle="yes">PeerJ.</italic>
</source>
                    <year>12 2018</year>;<volume>6</volume>:<fpage>e6034</fpage>&#x2013;<lpage>e6034</lpage>.
                    <pub-id pub-id-type="pmid">30564518</pub-id>
                    <pub-id pub-id-type="doi">10.7717/peerj.6034</pub-id>
                    <pub-id pub-id-type="pmcid">PMC6286809</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref11">
                <label>11</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Koslicki</surname>
                            <given-names>D</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Novak</surname>
                            <given-names>M</given-names>
                        </name>
</person-group>:
                    <article-title>Exact probabilities for the indeterminacy of complex networks as perceived through press perturbations.</article-title>
                    <source>

                        <italic toggle="yes">J. Math. Biol.</italic>
</source>
                    <year>2018</year>;<volume>76</volume>(<issue>4</issue>):<fpage>877</fpage>&#x2013;<lpage>909</lpage>.
                    <pub-id pub-id-type="pmid">28735343</pub-id>
                    <pub-id pub-id-type="doi">10.1007/s00285-017-1163-0</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref12">
                <label>12</label>
                <mixed-citation publication-type="book">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Levins</surname>
                            <given-names>R</given-names>
                        </name>
</person-group>:
                    <source>

                        <italic toggle="yes">Evolution in Changing Environments: Some Theoretical Explorations. Monographs in Population biology.</italic>
</source>
                    <publisher-loc>Princeton, N.J.</publisher-loc>:
                    <publisher-name>Princeton University Press</publisher-name>;<year>1968</year>.
                    <pub-id pub-id-type="doi">10.1515/9780691209418</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref13">
                <label>13</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Levins</surname>
                            <given-names>R</given-names>
                        </name>
</person-group>:
                    <article-title>The qualitative analysis of partially specified systems.</article-title>
                    <source>

                        <italic toggle="yes">Ann. N. Y. Acad. Sci.</italic>
</source>
                    <year>1974</year>;<volume>231</volume>:<fpage>123</fpage>&#x2013;<lpage>138</lpage>.
                    <pub-id pub-id-type="pmid">4522890</pub-id>
                    <pub-id pub-id-type="doi">10.1111/j.1749-6632.1974.tb20562.x</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref14">
                <label>14</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Melbourne-Thomas</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Wotherspoon</surname>
                            <given-names>S</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Raymond</surname>
                            <given-names>B</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Comprehensive evaluation of model uncertainty in qualitative network analyses.</article-title>
                    <source>

                        <italic toggle="yes">Ecol. Monogr.</italic>
</source>
                    <year>2012</year>;<volume>82</volume>(<issue>4</issue>):<fpage>505</fpage>&#x2013;<lpage>519</lpage>.
                    <pub-id pub-id-type="doi">10.1890/12-0207.1</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref15">
                <label>15</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Mueller</surname>
                            <given-names>LAJ</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kugler</surname>
                            <given-names>KG</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Dander</surname>
                            <given-names>A</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>QuACN: an R package for analyzing complex biological networks quantitatively.</article-title>
                    <source>

                        <italic toggle="yes">Bioinformatics.</italic>
</source>
                    <year>11 2010</year>;<volume>27</volume>(<issue>1</issue>):<fpage>140</fpage>&#x2013;<lpage>141</lpage>.
                    <pub-id pub-id-type="pmid">21075747</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btq606</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref16">
                <label>16</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Mark Novak</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Wootton</surname>
                            <given-names>T</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Doak</surname>
                            <given-names>DF</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Predicting community responses to perturbations in the face of imperfect knowledge and network complexity.</article-title>
                    <source>

                        <italic toggle="yes">Ecology.</italic>
</source>
                    <year>2011</year>;<volume>92</volume>(<issue>4</issue>):<fpage>836</fpage>&#x2013;<lpage>846</lpage>.
                    <pub-id pub-id-type="pmid">21661547</pub-id>
                    <pub-id pub-id-type="doi">10.1890/10-1354.1</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref17">
                <label>17</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Novak</surname>
                            <given-names>M</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Yeakel</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Noble</surname>
                            <given-names>AE</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Characterizing species interactions to understand press perturbations: What is the community matrix?.</article-title>
                    <source>

                        <italic toggle="yes">Annu. Rev. Ecol. Evol. Syst.</italic>
</source>
                    <year>2016</year>;<volume>47</volume>:<fpage>409</fpage>&#x2013;<lpage>432</lpage>.
                    <pub-id pub-id-type="doi">10.1146/annurev-ecolsys-032416-010215</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref18">
                <label>18</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Pomeranz</surname>
                            <given-names>JPF</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Thompson</surname>
                            <given-names>RM</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Poisot</surname>
                            <given-names>T</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Inferring predator&#x2013;prey interactions in food webs.</article-title>
                    <source>

                        <italic toggle="yes">Methods Ecol. Evol.</italic>
</source>
                    <year>2019/06/12 2019</year>;<volume>10</volume>(<issue>3</issue>):<fpage>356</fpage>&#x2013;<lpage>367</lpage>.
                    <pub-id pub-id-type="doi">10.1111/2041-210X.13125</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref19">
                <label>19</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Rajala</surname>
                            <given-names>M</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Ritala</surname>
                            <given-names>R</given-names>
                        </name>
</person-group>:
                    <article-title>Topology estimation method for telecommunication networks.</article-title>
                    <source>

                        <italic toggle="yes">Telecommun. Syst.</italic>
</source>
                    <year>2018</year>;<volume>68</volume>(<issue>4</issue>):<fpage>745</fpage>&#x2013;<lpage>759</lpage>.
                    <pub-id pub-id-type="doi">10.1007/s11235-018-0422-8</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref20">
                <label>20</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Rand</surname>
                            <given-names>DA</given-names>
                        </name>
</person-group>:
                    <article-title>Mapping global sensitivity of cellular network dynamics: sensitivity heat maps and a global summation law.</article-title>
                    <source>

                        <italic toggle="yes">J. R. Soc. Interface.</italic>
</source>
                    <year>2019/06/12 2008</year>;<volume>5</volume>(<issue>suppl_1</issue>):<fpage>S59</fpage>&#x2013;<lpage>S69</lpage>.</mixed-citation>
            </ref>
            <ref id="ref21">
                <label>21</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Rice</surname>
                            <given-names>JJ</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Yuhai</surname>
                            <given-names>T</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Stolovitzky</surname>
                            <given-names>G</given-names>
                        </name>
</person-group>:
                    <article-title>Reconstructing biological networks using conditional correlation analysis.</article-title>
                    <source>

                        <italic toggle="yes">Bioinformatics.</italic>
</source>
                    <year>6/12/2019 2004</year>;<volume>21</volume>(<issue>6</issue>):<fpage>765</fpage>&#x2013;<lpage>773</lpage>.
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/bti064</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref22">
                <label>22</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Schlauch</surname>
                            <given-names>D</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Paulson</surname>
                            <given-names>JN</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Young</surname>
                            <given-names>A</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Estimating
 gene regulatory networks with pandaR.</article-title>
                    <source>

                        <italic toggle="yes">Bioinformatics.</italic>
</source>
                    <year>03 2017</year>;<volume>33</volume>(<issue>14</issue>):<fpage>2232</fpage>&#x2013;<lpage>2234</lpage>.
                    <pub-id pub-id-type="pmid">28334344</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/btx139</pub-id>
                    <pub-id pub-id-type="pmcid">PMC5870629</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref23">
                <label>23</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Sontag</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kiyatkin</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kholodenko</surname>
                            <given-names>BN</given-names>
                        </name>
</person-group>:
                    <article-title>Inferring dynamic architecture of cellular networks using time series of gene expression, protein and metabolite data.</article-title>
                    <source>

                        <italic toggle="yes">Bioinformatics.</italic>
</source>
                    <year>2004</year>;<volume>20</volume>(<issue>12</issue>):<fpage>1877</fpage>&#x2013;<lpage>1886</lpage>.
                    <pub-id pub-id-type="pmid">15037511</pub-id>
                    <pub-id pub-id-type="doi">10.1093/bioinformatics/bth173</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref24">
                <label>24</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Rachel Wang</surname>
                            <given-names>YX</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Huang</surname>
                            <given-names>H</given-names>
                        </name>
</person-group>:
                    <article-title>Review on statistical methods for gene network reconstruction using expression data.</article-title>
                    <source>

                        <italic toggle="yes">J. Theor. Biol.</italic>
</source>
                    <year>2014</year>;<volume>362</volume>(0):<fpage>53</fpage>&#x2013;<lpage>61</lpage>.
                    <pub-id pub-id-type="doi">10.1016/j.jtbi.2014.03.040</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref25">
                <label>25</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Willing</surname>
                            <given-names>BP</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Russell</surname>
                            <given-names>SL</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Finlay</surname>
                            <given-names>BB</given-names>
                        </name>
</person-group>:
                    <article-title>Shifting the balance: antibiotic effects on host&#x2013;microbiota mutualism.</article-title>
                    <source>

                        <italic toggle="yes">Nat. Rev. Microbiol.</italic>
</source>
                    <year>02 2011</year>;<volume>9</volume>:<fpage>233</fpage>&#x2013;<lpage>243</lpage>.
                    <pub-id pub-id-type="doi">10.1038/nrmicro2536</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref26">
                <label>26</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Yodzis</surname>
                            <given-names>P</given-names>
                        </name>
</person-group>:
                    <article-title>The indeterminacy of ecological interactions as perceived through perturbation experiments.</article-title>
                    <source>

                        <italic toggle="yes">Ecology.</italic>
</source>
                    <year>1988</year>;<volume>69</volume>(<issue>2</issue>):<fpage>508</fpage>&#x2013;<lpage>515</lpage>.
                    <pub-id pub-id-type="doi">10.2307/1940449</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref27">
                <label>27</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Yodzis</surname>
                            <given-names>P</given-names>
                        </name>
</person-group>:
                    <article-title>Must top predators be culled for the sake of fisheries?.</article-title>
                    <source>

                        <italic toggle="yes">Trends Ecol. Evol.</italic>
</source>
                    <year>2001</year>;<volume>16</volume>(<issue>2</issue>):<fpage>78</fpage>&#x2013;<lpage>84</lpage>.
                    <pub-id pub-id-type="pmid">11165705</pub-id>
                    <pub-id pub-id-type="doi">10.1016/S0169-5347(00)02062-0</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref28">
                <label>28</label>
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Zamir</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bastiaens</surname>
                            <given-names>PIH</given-names>
                        </name>
</person-group>:
                    <article-title>Reverse engineering intracellular biochemical networks.</article-title>
                    <source>

                        <italic toggle="yes">Nat. Chem. Biol.</italic>
</source>
                    <year>2008</year>;<volume>4</volume>(<issue>11</issue>):<fpage>643</fpage>&#x2013;<lpage>647</lpage>.
                    <pub-id pub-id-type="pmid">18936743</pub-id>
                    <pub-id pub-id-type="doi">10.1038/nchembio1108-643</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref29">
                <label>29</label>
                <mixed-citation publication-type="other">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Koslicki</surname>
                            <given-names>D</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Novak</surname>
                            <given-names>M</given-names>
                        </name>
</person-group>:
                    <article-title>dkoslicki/PressPurt: v1.0.1 (v1.0.1).</article-title>
                    <source>

                        <italic toggle="yes">Zenodo.</italic>
</source>
                    <year>2021</year>.
                    <pub-id pub-id-type="doi">10.5281/zenodo.5661173</pub-id>
                </mixed-citation>
            </ref>
        </ref-list>
    </back>
</article>
