% !Rnw weave = knitr %%% DO NOT EDIT the .tex file directly since it is generated from the .Rnw %%% sources. %\VignetteEngine{knitr::knitr} %\VignetteIndexEntry{irace package: User Guide} %\VignetteDepends{knitr} %\VignetteCompiler{knitr} \synctex=1 \RequirePackage{xparse} \RequirePackage[dvipsnames]{xcolor} \documentclass[a4paper,english]{article} \usepackage[utf8]{inputenc} \usepackage[T1]{fontenc} \usepackage{lmodern} \usepackage[a4paper]{geometry} % It saves some pages \usepackage[english]{babel} \usepackage{ifthen} \newboolean{Release} \setboolean{Release}{true} \usepackage{calc} \usepackage{afterpage} \usepackage{algorithm,algorithmic} \usepackage{booktabs} \usepackage{tabularx} \usepackage{xspace} \usepackage{amsmath,amssymb} \usepackage{relsize} \usepackage{fancyvrb} \usepackage{underscore} \usepackage{microtype} % \texttt{test -- test} keeps the "--" as "--" (and does not convert it to an en dash) \DisableLigatures{encoding = T1, family = tt* } \usepackage[hyphens]{url} \usepackage{hyperref} \usepackage[numbers]{natbib} \usepackage[nottoc]{tocbibind} %% For autoref \hypersetup{ colorlinks, linkcolor={red!50!black}, citecolor={blue!50!black}, urlcolor={blue!70!black} } \addto\extrasenglish{% \def\sectionautorefname{Section} \let\subsectionautorefname\sectionautorefname \let\subsubsectionautorefname\sectionautorefname } \usepackage[titletoc, title]{appendix} % Fix use with \autoref \newcommand*{\Appendixautorefname}{Appendix} \usepackage{tocloft} \setlength{\cftsubsecnumwidth}{3em}% Set length of number width in ToC for \subsection \usepackage[inline]{enumitem} \setlist[enumerate]{leftmargin=*,widest=00} \setlist[itemize]{leftmargin=1.5em} %% FIXME: listing is very limited, we should use 'minted' \usepackage{listings} \lstdefinestyle{BashInputStyle}{ language=bash,% basicstyle=\ttfamily,% numbers=none,% frame=tb,% rulecolor=\color{lightgray}, % framesep=1ex, framexleftmargin=1ex, columns=fullflexible,% backgroundcolor=\color{yellow!05},% linewidth=\linewidth,% % xleftmargin=1\linewidth,% identifierstyle=\color{darkgray},% keywordstyle=\color{darkgray},% keywordstyle={[2]\color{Cyan}},% keywordstyle={[3]\color{olive}},% stringstyle=\color{MidnightBlue},% commentstyle=\color{RedOrange},% morestring=[b]',% showstringspaces=false } \DefineVerbatimEnvironment{Code}{Verbatim}{} \DefineVerbatimEnvironment{CodeInput}{Verbatim}{fontshape=rm} \DefineVerbatimEnvironment{CodeOutput}{Verbatim}{} \newenvironment{CodeChunk}{}{} \newcommand{\IRACEHOME}[1]{\hyperlink{irace_home}{\path{$IRACE_HOME}}\path{#1}} \providecommand{\keywords}[1]{\textbf{\textit{Index terms---}} #1} % Simple font selection is not good enough. For example, |\texttt{--}| % gives `\texttt{--}', i.e., an endash in typewriter font. Hence, we % need to turn off ligatures, which currently only happens for commands % |\code| and |\samp| and the ones derived from them. Hyphenation is % another issue; it should really be turned off inside |\samp|. And % most importantly, \LaTeX{} special characters are a nightmare. E.g., % one needs |\~{}| to produce a tilde in a file name marked by |\file|. % Perhaps a few years ago, most users would have agreed that this may be % unfortunate but should not be changed to ensure consistency. But with % the advent of the WWW and the need for getting `|~|' and `|#|' into % URLs, commands which only treat the escape and grouping characters % specially have gained acceptance \makeatletter \DeclareRobustCommand\code{\bgroup\@makeother\_\@makeother\~\@makeother\$\@noligs\@codex} \def\@codex#1{\texorpdfstring% {{\text{\normalfont\ttfamily\hyphenchar\font=-1 #1}}}% {#1}\egroup} \makeatother \let\proglang=\textsf \newcommand{\pkg}[1]{{\fontseries{b}\selectfont #1}} \newcommand{\aR}{\proglang{R}\xspace} \newcommand{\MATLAB}{\proglang{MATLAB}\xspace} \newcommand{\eg}{e.g.,\xspace} \newcommand{\SoftwarePackage}{\pkg} \newcommand{\ACOTSP}{\SoftwarePackage{ACOTSP}\xspace} %% How to use this command: % Parameter with one short switch: \defparameter[short]{paramName}{long}{default} % Parameter without short switch: \defparameter{paramName}{long}{default} % Parameter without switch: \defparameter{paramName}{}{default} \newcommand{\defparameter}[4][]{% \item[\code{#2}]\hypertarget{opt:#2}{} ~~ % \ifthenelse{\equal{#3}{}}{}{% \emph{flag:} % \ifthenelse{\equal{#1}{}}{}{% \code{-#1}~~~\emph{or}~~~}% \code{--#3} ~~ }% \emph{default:}~\texttt{#4} \\ } \newcommand{\parameter}[1]{\hyperlink{opt:#1}{\code{#1}}} \newcommand{\iracefun}[1]{\href{https://mlopez-ibanez.github.io/irace/reference/#1.html}{\code{#1()}}} %\usepackage{showlabels} %\showlabels{hypertarget} \newcommand{\irace}{\pkg{irace}\xspace} \newcommand{\Irace}{\pkg{Irace}\xspace} \newcommand{\race}{\pkg{race}\xspace} \newcommand{\FRACE}{\text{F-Race}\xspace} \newcommand{\IFRACE}{\text{I/F-Race}\xspace} \newcommand{\PyImp}{\pkg{PyImp}\xspace} \newcommand{\iraceversion}{\Sexpr{packageVersion("irace")}} \newcommand{\Niter}{\ensuremath{N^\text{iter}}\xspace} \newcommand{\Nparam}{\ensuremath{{N^\text{param}}}\xspace} \newcommand{\iter}{\ensuremath{j}\xspace} \newcommand{\Budget}{\ensuremath{B}\xspace} \newcommand{\Budgetj}{\ensuremath{\Budget_{\iter}}\xspace} \newcommand{\Bused}{\ensuremath{\Budget_\text{used}}\xspace} \newcommand{\Ncand}[1][]{\ensuremath{N_{#1}}\xspace} \newcommand{\Mui}{\ensuremath{\mu_{\iter}}\xspace} \newcommand{\Nmin}{\ensuremath{N^\text{min}}\xspace} \newcommand{\Nsurv}{\ensuremath{N^\text{surv}}\xspace} \newcommand{\Nelite}{\ensuremath{N^\text{elite}}\xspace} \newcommand{\Nnew}{\ensuremath{N^\text{new}}\xspace} \newcommand{\bmax}{\ensuremath{b^\text{max}}\xspace} \newcommand{\bmin}{\ensuremath{b^\text{min}}\xspace} \newcommand{\Celite}{\ensuremath{\Theta^\text{elite}}\xspace} \ifthenelse {\boolean{Release}}{% \newcommand{\MANUEL}[1]{} \newcommand{\LESLIE}[1]{} \newcommand{\THOMAS}[1]{} }{% \newcommand{\MANUEL}[1]{{\footnotesize\noindent\textbf{\color{red}[~MANUEL: #1~]}}} \newcommand{\LESLIE}[1]{\footnote{\noindent\textbf{[ LESLIE: #1 ]}}} \newcommand{\THOMAS}[1]{\footnote{\noindent\textbf{[ THOMAS: #1 ]}}} } \newcommand{\hide}[1]{} \usepackage{tcolorbox} \newcommand{\infoicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{light-bulb-icon}}% \hspace{1em}} \newcommand{\warningicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{Warning-icon}}% \hspace{1em}} \definecolor{LightGray}{RGB}{193,193,193} \definecolor{LightYellow}{RGB}{253,247,172} \newlength\macroiconwidth \newenvironment{xwarningbox}{% \setlength{\fboxrule}{3.0\fboxrule}% \setlength{\fboxsep}{0\fboxsep}% \begin{tcolorbox}[colback=LightYellow,colframe=LightGray,boxrule=\fboxrule,boxsep=\fboxsep]% \infoicon% \settowidth{\macroiconwidth}{\infoicon}% \begin{minipage}[c]{\columnwidth - \macroiconwidth - 2.0\fboxrule - 2.0\fboxsep} \raggedright\footnotesize % }{% \end{minipage} \end{tcolorbox} % } % Workaround for broken knitr: https://github.com/yihui/knitr-examples/blob/master/036-latex-if.tex \providecommand{\hldef}[1]{\textcolor[rgb]{0.345,0.345,0.345}{#1}}% \providecommand{\hlsng}[1]{\textcolor[rgb]{0.192,0.494,0.8}{#1}}% <<setup, include=FALSE>>= library(knitr) knit_hooks$set(inline = function(x) { if (is.numeric(x)) return(knitr:::format_sci(x, 'latex')) highr::hi_latex(x) }) @ \begin{document} <<include=FALSE>>= library(knitr) @ \author{Manuel L\'opez-Ib\'a\~nez, Leslie P\'erez C\'aceres, J\'er\'emie Dubois-Lacoste,\\ Thomas St\"utzle and Mauro Birattari \\IRIDIA, CoDE, Universit\'e Libre de Bruxelles, Brussels, Belgium} \title{The \irace Package: User Guide} \date{Version \iraceversion, \today} %\keywords{automatic % algorithm configuration, racing, parameter tuning, \aR} \maketitle \tableofcontents %Load files needed for examples <<exampleload,eval=TRUE,include=FALSE>>= library("irace") load("examples.Rdata") # loads "experiment" and "output" iraceResults <- irace::read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)) log_ablation_file <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork = TRUE) load(log_ablation_file) options(width = 70) @ \newpage %% %% %% %% General info %% %% %% \section{General information} \MANUEL{Some things could be taken from the intro of the irace paper and reformulated.} \MANUEL{It would be good to mention that not only opt algorithms can be configured with irace, we say this in the paper.} \subsection{Background} \MANUEL{I would add a paragraph defining what is irace (a bit longer than the abstract above) and references to the literature so people can find more info. The first reference should be the irace TR.} \LESLIE{Here i guess we should say why tune an algorithm is a good idea, and why using irace is a better one.} The \irace package implements an \emph{iterated racing} procedure, which is an extension of Iterated F-race (\IFRACE)~\cite{BirYuaBal2010:emaoa}. The main use of \irace is the automatic configuration of optimization and decision algorithms, that is, finding the most appropriate settings of an algorithm given a set of instances of a problem. However, it may also be useful for configuring other types of algorithms when performance depends on the used parameter settings. It builds upon the \pkg{race} package by Birattari and it is implemented in \aR. The \irace package is available from CRAN: % \begin{center} \url{https://cran.r-project.org/package=irace} \end{center} % More information about \irace is available at \url{https://mlopez-ibanez.github.io/irace}. \subsection{Version} The current version of the \irace package is \iraceversion. Previous versions of the package can also be found in the \href{https://cran.r-project.org/package=irace}{CRAN website}. The algorithm underlying the current version of \irace and its motivation are described by \citet{LopDubPerStuBir2016irace}. The \textbf{adaptive capping mechanism} available from version $3.0$ is described by \citet{PerLopHooStu2017:lion}. Details of the implementation before version 2.0 can be found in a previous technical report~\cite{LopDubStu2011irace}. % \begin{xwarningbox} Versions of \irace before 2.0 are not compatible with the file formats detailed in this document. \end{xwarningbox} \subsection{License} The \irace package is Copyright \copyright{} \the\year\ and distributed under the GNU General Public License version 3.0 (\url{http://www.gnu.org/licenses/gpl-3.0.en.html}). The \irace package is free software (software libre): You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The \irace package is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. Please be aware that the fact that this program is released as Free Software does not excuse you from scientific propriety, which obligates you to give appropriate credit! If you write a scientific paper describing research that made substantive use of this program, it is your obligation as a scientist to (a) mention the fashion in which this software was used in the Methods section; (b) mention the algorithm in the References section. The appropriate citation is: \begin{itemize}[leftmargin=3em] \item[] Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Leslie Pérez Cáceres, Thomas Stützle, and Mauro Birattari. The \irace package: Iterated Racing for Automatic Algorithm Configuration. \emph{Operations Research Perspectives}, 3:43--58, 2016. doi:~\href{http://dx.doi.org/10.1016/j.orp.2016.09.002}{10.1016/j.orp.2016.09.002} \end{itemize} \section{Before starting} \MANUEL{I think this could be a bit more detailed by defining what is a parameter, a configuration, an instance, etc. but ok for now.} The \irace package provides an automatic configuration tool for tuning optimization algorithms, that is, automatically finding good configurations for the parameters values of a (target) algorithm saving the effort that normally requires manual tuning. \begin{figure}[t] \centering \includegraphics[width=0.6\textwidth]{irace-scheme} \caption{Scheme of \irace flow of information.} \label{fig:irace-scheme} \end{figure} Figure~\ref{fig:irace-scheme} gives a general scheme of how \irace works. \Irace receives as input a \emph{parameter space definition} corresponding to the parameters of the target algorithm that will be tuned, a set of \emph{instances} for which the parameters must be tuned for and a set of options for \irace that define the \emph{configuration scenario}. Then, \irace searches in the parameter search space for good performing algorithm configurations by executing the target algorithm on different instances and with different parameter configurations. A \parameter{targetRunner} must be provided to execute the target algorithm with a specific parameter configuration ($\theta$) and instance ($i$). The \parameter{targetRunner} function (or program) acts as an interface between the execution of the target algorithm and \irace: It receives the instance and configuration as arguments and must return the evaluation of the execution of the target algorithm. The following user guide contains guidelines for installing \irace, defining configuration scenarios, and using \irace to automatically configure your algorithms. %% %% %% %% Installation %% %% %% \section{Installation} \subsection{System requirements} \begin{itemize} \item \aR ($\text{version} \geq 3.2.0$) is required for running irace, but you don't need to know the \aR language to use it. \aR is freely available and you can download it from the \aR project website (\url{https://www.r-project.org}). See \autoref{sec:installation} for a quick installation guide of \aR. \item For GNU/Linux and OS X, the command-line executable \code{parallel-irace} requires GNU Bash. Individual examples may require additional software. \end{itemize} \subsection{\irace installation} \label{sec:irace install} The \irace package can be installed automatically within \aR or by manual download and installation. We advise to use the automatic installation unless particular circumstances do not allow it. The instructions to install \irace with the two mentioned methods are the following: \subsubsection[Install automatically within R]{Install automatically within \aR{}} Execute the following line in the \aR console to install the package: <<R_irace_install, prompt=FALSE, eval=FALSE>>= install.packages("irace") @ Select a mirror close to your location, and test the installation in the \aR console with: <<R_irace_launch,eval=FALSE, prompt=FALSE>>= library("irace") q() # To exit R @ Alternatively, within the \aR graphical interface, you may use the \code{Packages and data->Package installer} menu on OS X or the \code{Packages} menu on Windows. \subsubsection{Manual download and installation} From the \irace package CRAN website (\url{https://cran.r-project.org/package=irace}), download one of the three versions available depending on your operating system: \begin{itemize} \item \code{irace_\iraceversion.tar.gz} (Unix/BSD/GNU/Linux) \item \code{irace_\iraceversion.tgz} (OS X) \item \code{irace_\iraceversion.zip} (Windows) \end{itemize} To install the package on GNU/Linux and OS X, you must execute the following command at the shell (replace \code{<package>} with the path to the downloaded file, either \code{irace_\iraceversion.tar.gz} or \code{irace_\iraceversion.zip}): % \begin{lstlisting}[style=BashInputStyle] R CMD INSTALL <package> \end{lstlisting} To install the package on Windows, open \aR and execute the following line on the \aR console (replace \code{<package>} with the path to the downloaded file \code{irace_\iraceversion.zip}): %\LESLIE{Check that this actually works on internet says that this: \code{Rscript -e "install.packages('foo.zip', repos = NULL)"} also works} % <<install_win1,eval=FALSE, prompt=FALSE>>= install.packages("<package>", repos = NULL) @ If the previous installation instructions fail because of insufficient permissions and you do not have sufficient admin rights to install \irace system-wide, then you need to force a local installation. \subsubsection{Local installation} Let's assume you wish to install \irace on a path denoted by \code{<R_LIBS_USER>}, which is a filesystem path for which you have sufficient rights. This directory \textbf{must} exist before attempting the installation. Moreover, you must provide to \aR the path to this library when loading the package. However, the latter can be avoided by adding the path to the system variable \code{R_LIBS} or to the \aR internal variable \code{.libPaths}, as we will see below.\footnote{% On Windows, see also \url{https://cran.r-project.org/bin/windows/base/rw-FAQ.html\#I-don_0027t-have-permission-to-write-to-the-R_002d3_002e3_002e1_005clibrary-directory}.} On GNU/Linux or OS X, execute the following commands to install the package on a local directory: \begin{lstlisting}[style=BashInputStyle] export R_LIBS_USER="<R_LIBS_USER>" # Create R_LIBS_USER if it doesn't exist mkdir $R_LIBS_USER # Replace <package> with the path to the downloaded file. R CMD INSTALL --library=$R_LIBS_USER <package> # Tell R where to find R_LIBS_USER export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} On Windows, you can install the package on a local directory by executing the following lines in the \aR console: <<install_win2,eval=FALSE, prompt=FALSE>>= # Replace <package> with the path to the downloaded file. # Replace <R_LIBS_USER> with the path used for installation. install.packages("<package>", repos = NULL, lib = "<R_LIBS_USER>") # Tell R where to find R_LIBS_USER. # This must be executed for every new session. .libPaths(c("<R_LIBS_USER>", .libPaths())) @ \subsubsection{Testing the installation and invoking irace} Once \irace has been installed, load the package and test that the installation was successful by opening an \aR console and executing: <<R_irace_test1, prompt=FALSE, eval=FALSE>>= # Load the package library("irace") # Obtain the installation path system.file(package = "irace") @ The last command must print out the filesystem path where \irace is installed. In the remainder of this guide, the variable \code{\$IRACE_HOME} is used to denote this path. When executing any provided command that includes the \code{\$IRACE_HOME} variable do not forget to replace this variable with the installation path of \irace. On GNU/Linux or OS X, you can let the operating system know where to find \irace by defining the \code{\$IRACE_HOME} variable and adding it to the system \code{PATH}. Append the following commands to \path{~/.bash_profile}, \path{~/.bashrc} or \path{~/.profile}: % %<<linux_irace_path1,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] # Replace <IRACE_HOME> with the irace installation path export IRACE_HOME=<IRACE_HOME> export PATH=${IRACE_HOME}/bin/:$PATH # Tell R where to find R_LIBS_USER # Use the following line only if local installation was forced export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} %@ Then, open a new terminal and launch \irace as follows: %<<linux_irace_help,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] irace --help \end{lstlisting} %@ On Windows, you need to add both \aR and the installation path of \irace to the environment variable \code{PATH}. To edit the \code{PATH}, search for ``Environment variables'' in the control panel, edit \code{PATH} and add a string similar to \path{C:\R_PATH\bin;C:\IRACE_HOME\bin\x64\} where \code{R_PATH} is the installation path of \aR and \code{IRACE_HOME} is the installation path of \irace. If \irace was installed locally, you also need to edit the environment variable \code{R_LIBS} to add \code{R_LIBS_USER}. Then, open a new terminal (run program \code{cmd.exe}) and launch \irace as: % %<<win_irace_help,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] irace.exe --help \end{lstlisting} %@ Alternatively, you may directly invoke \irace from within the \aR console by executing: <<windows_irace_help ,eval=FALSE, prompt=FALSE>>= library("irace") irace_cmdline("--help") @ \section{Running irace}\label{sec:execution} Before performing the tuning of your algorithm, it is necessary to define a tuning scenario that will give \irace all the necessary information to optimize the parameters of the algorithm. The tuning scenario is composed of the following elements: \begin{enumerate} \item Target algorithm parameter description (see \autoref{sec:target parameters}). \item Target algorithm runner (see \autoref{sec:runner}). \item Training instances list (see \autoref{sec:training}) \item \irace options (see \autoref{sec:irace options}). \item \textit{Optional:} Initial configurations (see \autoref{sec:initial}). \item \textit{Optional:} Target algorithm evaluator (see \autoref{sec:evaluator}). \end{enumerate} These scenario elements can be provided as plain text files or as \aR objects. This user guide provides examples of both types, but we advise the use of plain text files, which we consider the simpler option. For a step-by-step guide to create the scenario elements for your target algorithm continue to \autoref{sec:step}. For an example execution of \irace using the \ACOTSP scenario go to \autoref{sec:example}. \subsection{Step-by-step setup guide}\label{sec:step} This section provides a guide to setup a basic execution of \irace. The template files provided in the package (\IRACEHOME{/templates}) will be used as basis for creating your new scenario. Please follow carefully the indications provided in each step and in the template files used; if you have doubts check the the sections that describe each option in detail. \begin{enumerate}[leftmargin=*] \item Create a directory (\eg~\path{./tuning/}) for the scenario setup. This directory will contain all the files that describe the scenario. On GNU/Linux or OS X, you can do this as follows: %<<dir1,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning \end{lstlisting} %@ \item Initialize the tuning directory with template config files. On GNU/Linux or OS X, you can do this as follows: %<<dir1,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] irace --init \end{lstlisting} %@ \item Define the target algorithm parameters to be tuned by following the instructions in \code{parameters.txt}. Available parameter types and other guidelines can be found in \autoref{sec:target parameters}. \item \textit{Optional}: Define the initial parameter configuration(s) of your algorithm, which allows you to provide good starting configurations (if you know some) for the tuning. Follow the instructions in \code{configurations.txt} and set \parameter{configurationsFile}\code{="configurations.txt"} in \code{scenario.txt}. More information in \autoref{sec:initial}. If you do not need to define initial configurations remove this file from the directory. \item Place the instances you would like to use for the tuning of your algorithm in the folder \path{./tuning/Instances/}. In addition, you can create a file (\eg~\code{instances-list.txt}) that specifies which instances from that directory should be run and which instance-specific parameters to use. To use such an instance file, set the appropriate option in \code{scenario.txt}, e.g., \parameter{trainInstancesFile} \code{= "instances-list.txt"}. See \autoref{sec:training} for guidelines. \item Uncomment and assign in \code{scenario.txt} only the options for which you need a value different from the default. Some common options that you might want to adjust are: \begin{description} \item[\parameter{execDir}] (\code{--exec-dir}): the directory in which \irace will execute the target algorithm; the default value is the current directory. %\item[\parameter{logFile}] (\code{--log-file}): a file the name of the results \aR data file that produces \irace. \item[\parameter{maxExperiments}] (\code{--max-experiments}): the maximum number of executions of the target algorithm that \irace will perform. \item[\parameter{maxTime}] (\code{--max-time}): maximum total execution time in seconds for the executions of \code{targetRunner}. In this case, \code{targetRunner} must return two values: cost and time. Note that you must provide either \parameter{maxTime} or \parameter{maxExperiments}. \item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set to \path{./Instances} if you put the training instances in that folder as instructed above. \end{description} For setting the tuning budget, see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}. \item Modify the \code{target-runner} script to run your algorithm. This script must execute your algorithm with the parameters and instance specified by \irace and return the evaluation of the execution and \textit{optionally} the execution time (\code{cost [time]}). When the \parameter{maxTime} option is used, returning \code{time} is mandatory. The \code{target-runner} template is written in \proglang{GNU Bash} scripting language, which can be executed easily in GNU/Linux and OS X systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. An example using Julia is available at \url{https://github.com/sbomsdorf/An-example-of-irace-using-Julia-code}. % Follow these instructions to adjust the given \code{target-runner} template to your algorithm: \begin{enumerate} \item Set the \code{EXE} variable with the path to the executable of the target algorithm. \item Set the \code{FIXED_PARAMS} if you need extra arguments in the execution line of your algorithm. An example could be the time that your algorithm is required to run (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--time 60"}) or the number of evaluations required (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--evaluations 10000"}). \item The line provided in the template executes the executable described in the \code{EXE} variable. % \begin{center} \code{\$EXE \$\{FIXED_PARAMS\} -i \$\{INSTANCE\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} % You must change this line according to the way your algorithm is executed. In this example, the algorithm receives the instance to solve with the flag \code{-i} and the seed of the random number generator with the flag \code{--seed}. The variable \code{CONFIG_PARAMS} adds to the command line the parameters that \irace has given for the execution. You must set the command line execution as needed. For example, the instance might not need a flag and might need to be the first argument: \begin{center} \code{\$EXE \$\{INSTANCE\} \$\{FIXED_PARAMS\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} The output of your algorithm is saved to the file defined in the \code{\$STDOUT} variable, and error output is saved in the file given by \code{\$STDERR}. The line: \begin{center} \code{if [ -s "\${STDOUT}" ]; then} \end{center} checks if the file containing the output of your algorithm is not empty. The example provided in the template assumes that your algorithm prints in the last output line the best result found (only a number). The line: \begin{center} \code{COST=\$(cat \$\{STDOUT\} | grep -e '\^{}[[:space:]]*[+-]\textbackslash{}?[0-9]' | cut -f1)} \end{center} parses the output of your algorithm to obtain the result from the last line. The \code{target-runner} script must print \textbf{only} one number. In the template example, the result is printed with \code{echo "\$COST"} (assuming \parameter{maxExperiments} is used) and the generated files are deleted (you may remove that line if you wish to keep them). \begin{xwarningbox} The \code{target-runner} script must be an executable file, unless you specify \parameter{targetRunnerLauncher}. \end{xwarningbox} You can test the target runner from the \aR console by checking the scenario as explained earlier in \autoref{sec:execution}. If you have problems related to the \code{target-runner} script when executing \irace, see \autoref{sec:check list} for a check list to help diagnose common problems. For more information about the \parameter{targetRunner}, please see \autoref{sec:runner}, \end{enumerate} \item \textit{Optional}: Modify the \code{target-evaluator} file. This is rarely needed and the \code{target-runner} template does not use it. \autoref{sec:evaluator} explains when a \parameter{targetEvaluator} is needed and how to define it. \item The \irace executable provides an option (\parameter{--check}) to check that the scenario is correctly defined. We recommend to perform a check every time you create a new scenario. When performing the check, \irace will verify that the scenario and parameter definitions are correct and will test the execution of the target algorithm. To check your scenario execute the following commands: \begin{itemize} \item From the command-line (on Windows, execute \code{irace.bat}): %<<irace_check,engine='bash', eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace --scenario scenario.txt --check \end{lstlisting} %@ \item Or from the \aR console: <<irace_R_check, eval=FALSE, prompt=FALSE>>= library("irace") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) checkIraceScenario(scenario = scenario) @ \end{itemize} \item Once all the scenario elements are prepared you can execute \irace, either using the command-line wrappers provided by the package or directly from the \aR console: \begin{itemize} \item \textbf{From the command-line console}, call the command (on Windows, you should execute \code{irace.exe}): \begin{lstlisting}[style=BashInputStyle] cd ./tuning/ # $IRACE_HOME is the installation directory of irace # By default, irace reads scenario.txt, you can specify a different file # with --scenario. $IRACE_HOME/bin/irace \end{lstlisting} For this example, we assume that the needed scenario files have been set properly in the \code{scenario.txt} file using the options described in \autoref{sec:irace options}. Most \irace options can be specified in the command line or directly in the \code{scenario.txt} file. \item \textbf{From the \aR console}, evaluate: <<irace_R_exe, eval=FALSE, prompt=FALSE>>= library("irace") # Go to the directory containing the scenario files setwd("./tuning") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) irace_main(scenario = scenario) @ \end{itemize} This will perform one run of \irace. See the output of \code{irace --help} in the command-line or \code{irace_cmdline("--help")} in \aR for quick information on additional \irace options. For more information about \irace options, see \autoref{sec:irace options}. \end{enumerate} \begin{xwarningbox} Command-line options override the same options specified in the \code{scenario.txt} file. \end{xwarningbox} \subsection{Setup example for ACOTSP}\label{sec:example} The \ACOTSP tuning example can be found in the package installation in the folder \IRACEHOME{/examples/acotsp}. % Other example scenarios can be found in the same folder. More examples of tuning scenarios can be found in the Algorithm Configuration Library (AClib, \url{http://www.aclib.net/}). In this section, we describe how to execute the \ACOTSP scenario. If you wish to start setting up your own scenario, continue to the next section. For this example, we assume a GNU/Linux system such as Ubuntu with a working \proglang{C} compiler such as \code{gcc}. To execute this scenario follow these steps: \begin{enumerate} \item Create a directory for the tuning (\eg~\path{./tuning/}) and copy the example scenario files located in the \code{examples} folder to the created directory: %<<dir0,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning # $IRACE_HOME is the installation directory of irace. cp $IRACE_HOME/examples/acotsp/* ./ ls ./ # Make sure that target-runner is executable chmod u+x target-runner \end{lstlisting} %@ \item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz} to the \path{./tuning/} directory and decompress it, which creates create a folder \path{instances}: %<<instance0,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] tar -xvf instances.tar.gz ls instances/ \end{lstlisting} %@ If the above gives an error or does not show any files, then the files were not extract correctly. Maybe the \path{instances.tar.gz} file did not download correctly or maybe it is not in the correct place. It should be within the folder \path{tuning}.correctly. \item Download the \ACOTSP software from \url{https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip} to the \path{./tuning/} directory and compile the \path{acotsp} executable using \code{make}. %<<acotsp0,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] unzip master.zip make -C ACOTSPQAP-master acotsp ./ACOTSPQAP-master/acotsp --version \end{lstlisting} %@ If the above gives an error, then the \path{acotsp} executable failed to compile for some reason. Maybe you are missing the C compiler or some files did not extract correctly. \item Create a directory for executing the experiments and execute \irace: %<<runexample,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] mkdir ./acotsp-arena/ # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace \end{lstlisting} %@ Or you can also execute \irace from the \aR console using: <<runexample2,prompt=FALSE,eval=FALSE>>= library("irace") setwd("./tuning/") irace_cmdline() @ \end{enumerate} The most usual sources of error when running the above commands are: \begin{itemize} \item The \irace package is not correctly installed. Please make sure that installing \irace did not give any errors. \item The location of the files is not correct. Please make sure that you have: \begin{itemize} \item The folder \path{tuning} and that it contains the files \path{scenario.txt}, \path{parameters-acotsp.txt}, \path{target-runner}, and the folders \path{instances}, \path{ACOTSPQAP-master} and \path{acotsp-arena}. \item The folder \path{instances} should contain the TSP instance files (\path{*.tsp}). \item The folder \path{ACOTSPQAP-master} should contain the executable \path{acotsp}. You should be able to invoke \code{./ACOTSPQAP-master/acotsp --version} without an error. \item The folder \path{acotsp-arena} should be empty. \end{itemize} \end{itemize} %% %% %% %% Scenario settings %% %% %% \section{Defining a configuration scenario}\label{sec:scenario} \subsection{Target algorithm parameters} \label{sec:target parameters} The parameters of the target algorithm are defined by a parameter file as described in \autoref{sec:parameters file}. Optionally, when executing \irace from the \aR console, the parameters can be specified directly as an \aR object (see \autoref{sec:parameters object}). For defining your parameters follow the guidelines provided in the following sections. \subsubsection{Parameter types} Each target parameter has an associated type that defines its domain and the way \irace handles them internally. Understanding the nature of the domains of the target parameters is important to select appropriate types. The four basic types supported by \irace are the following: \begin{itemize} \item \textit{Real} parameters are numerical parameters that can take floating-point values within a given range. The range is specified as an interval `\code{(<lower bound>,<upper bound>)}'. This interval is closed, that is, the parameter value may eventually be one of the bounds. The possible values are rounded to a number of \emph{decimal places} specified by the global option \code{digits} (Section~\ref{sec:globaloptions}). For example, given the default number of digits of $4$, the values $0.12345$ and $0.12341$ are both rounded to $0.1234$. Selected real-valued parameters can be optionally sampled on a logarithmic scale (base $e$). \item \textit{Integer} parameters are numerical parameters that can take only integer values within the given range. Their range is specified as the range of real parameters and they can also be optionally sampled on a logarithmic scale (base $e$). \item \textit{Categorical} parameters are defined by a set of possible values specified as `\code{(<value 1>,} \code{...,} \code{<value n>)}'. The values are quoted or unquoted character strings. Empty strings and strings containing commas or spaces must be quoted. \item \emph{Ordinal} parameters are defined by an \emph{ordered} set of possible values in the same format as for categorical parameters. They are handled internally as integer parameters, where the integers correspond to the indexes of the values. \end{itemize} \begin{xwarningbox} Boolean (or logical) parameters are best encoded as categorical ones with just two values rather than integer ones with domain $(0,1)$. Some boolean parameters take an explicit value (0/1 or true/false) such as: \begin{CodeInput} dlb "--dlb " c (0, 1) \end{CodeInput} Others are switches whose presence activates the parameter: \begin{CodeInput} dlb "" c ("", "--dlb") \end{CodeInput} \end{xwarningbox} \subsubsection{Parameter domains} For each target parameter, an interval or a set of values must be defined according to its type, as described above. There is no limit for the size of the set or the length of the interval, but keep in mind that larger domains could increase the difficulty of the tuning task. Choose always values that you consider relevant for the tuning. In case of doubt, we recommend to choose larger intervals, as occasionally best parameter settings may be not intuitive a priori. All intervals are considered as closed intervals. It is possible to define parameters that will have always the same value. Such ``\emph{fixed}'' parameters will not be tuned but their values are used when executing the target algorithm and they are affected by constraints defined on them. All fixed parameters must be defined as categorical parameters and have a domain of one element. \subsubsection{Parameter dependent domains}\label{sec:paramdependant} Domains that are dependent on the values of other parameters can be specified only for numerical parameters (both integer and real). To do so, the dependent domain must be expressed in function of another parameter, which must be a numerical parameter. The expression that defines a dependency must be written between quotes: \code{(value, "expression")} or \code{("expression", value)} or \code{("expression", "expression")}. The expressions can only use the following operators and \aR functions: \code{+}, \code{-}, \code{*}, \code{/}, \code{\%\%}, \code{min}, \code{max}, \code{round}, \code{floor}, \code{ceiling}, \code{trunc}. If you need to use an operator or function not listed here, please contact us. \begin{xwarningbox} The user must ensure that the defined domain is valid at all times since \irace currently is not able to detect possible invalid domains based on the expressions provided. \end{xwarningbox} If you have a parameter \code{p2} that is just a transformation of another \code{p1}, then instead of using a dependent domain (left-hand side of the following example), it will be better to create a dummy parameter that controls the transformation (right-hand side) and do the transformation within \code{target-runner}. For example: % \begin{center} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With dependent domains p1 "" r (0, 100) p2 "" r ("p1", "p1 + 10") \end{CodeInput} \end{minipage} \hspace{\stretch{1}} should be \hspace{\stretch{1}} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With a dummy parameter p1 "" r (0, 100) p2dum "" r (0, 10) \end{CodeInput} \end{minipage} \end{center} % and \code{target-runner} will compute $\code{p2} = \code{p2dum} \cdot \code{p1}$. \subsubsection{Conditional parameters}\label{sec:conditional} Conditional parameters are active only when others have certain values. These dependencies define a hierarchical relation between parameters. For example, the target algorithm may have a parameter \code{localsearch} that takes values \code{(sa, ts)} and another parameter \code{ts-length} that only needs to be set if the first parameter takes precisely the value \code{ts}. Thus, parameter \code{ts-length} is conditional on \code{localsearch == "ts"}. \subsubsection{Forbidden parameter configurations}\label{sec:forbidden} A line containing just \texttt{[forbidden]} ends the list of parameters and starts the list of forbidden expressions. Each line is a logical expression (in \aR syntax) containing parameter names as defined by the \parameter{parameterFile} (\autoref{sec:target parameters}), values and logical operators. For a list of \aR logical operators see: \begin{center} \url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html} \end{center} If \irace generates a parameter configuration that makes any of the logical expressions evaluate to \code{TRUE}, then the configuration is considered forbidden and it is never evaluated. This is useful when some combination of parameter values could cause the target algorithm to crash, consume excessive CPU time or memory, or when it is known that they do no produce satisfactory results. \begin{xwarningbox} Initial configuration (\autoref{sec:initial}) that are forbidden will be discarded with a warning. \end{xwarningbox} If the forbidden constraints provided are too strict, \irace may produce the following error: \begin{CodeInput} irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict? \end{CodeInput} % In that case, it may be a good idea to reformulate the forbidden constraints as conditional parameters (\autoref{sec:conditional}), parameter-dependent domains (\autoref{sec:paramdependant}), repairing the configurations (\autoref{sec:repairconf}) or post-processing within the target-algorithm (\autoref{sec:complex_domains}). \subsubsection{Global options}\label{sec:globaloptions} A line containing just \texttt{[global]} starts the definition of global options. The only global option currently implemented is \code{digits}, which controls the number of decimal digits for real valued parameters. Its default value is 4. \subsubsection{Parameter file format}\label{sec:parameters file} For simplicity, the description of the parameters space is given as a table. Each line of the table defines a configurable parameter % \begin{center} \code{<name>\ <label>\ <type>\ <domain>\ [ | <condition>\ ] } \end{center} where each field is defined as follows: % \begin{description}[left=5em,itemindent=-0.5em,align=right,itemsep=0pt] % \DrawEnumitemLabel \item[\code{<name>}] The name of the parameter as an unquoted alphanumeric string, e.g., `\code{ants}'. % \item[\code{<label>}] A \emph{label} for this parameter. This is a string that will be passed together with the parameter to \parameter{targetRunner}. In the default \parameter{targetRunner} provided with the package (\autoref{sec:runner}), this is the command-line switch used to pass the value of this parameter, for instance `\code{"--ants "}'.\newline The value of the parameter is concatenated \emph{without separator} to the label when invoking \parameter{targetRunner}, thus \emph{any whitespace in the label is significant}. Following the same example, when parameter \code{ants} takes value \code{5}, the default targetRunner will pass the parameter as \code{"--ants 5"}. \item[\code{{<type>}}] The type of the parameter, either \textit{integer}, \textit{real}, \textit{ordinal} or \textit{categorical}, given as a single letter: `\code{i}', `\code{r}', `\code{o}' or `\code{c}'. Numerical parameters can be sampled using a natural logarithmic scale with '\code{i,log}' and '\code{r,log}' (without spaces) for integer and real parameters, respectively. % \item[\code{{<domain>}}] The range or set of values of the parameter delimited by parentheses, \eg~\code{(0,1)} or \code{(a,b,c,d)}. See also parameter dependent domains (\autoref{sec:paramdependant}). % \item[\code{{<condition>}}] An optional \emph{condition} that determines whether the parameter is enabled or disabled, thus making the parameter conditional. If the condition evaluates to false, then no value is assigned to this parameter, and neither the parameter value nor the corresponding label are passed to \parameter{targetRunner}. The condition must follow the same syntax as those for specifying forbidden configurations (see below), that is, it must be a valid \aR logical expression\footnote{For a list of \aR operators see: \url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html}}. The condition may contain the name of other parameters as long as the dependency graph does not contain any cycle. Otherwise, \irace will detect the cycle and stop with an error. \end{description} \begin{xwarningbox} Categorical and ordinal parameters are always treated as strings. Given a parameter like: \begin{CodeInput} a "" c (0, 5, 10, 20) \end{CodeInput} then, a condition like \code{a > 10} will be true when \code{a} is \code{5}, because comparisons between strings are lexicographic and \code{"10"} is sorted before \code{"5"}. As a work-around, you can convert the string to numeric in the condition with \code{as.numeric(a)}. \end{xwarningbox} As an example, Figure~\ref{fig:acotsp_parameters} shows the parameters file of the \ACOTSP scenario. \begin{figure}[!hbt] \small\centering% \begin{CodeInput} # name switch type values [conditions (using R syntax)] algorithm "--" c (as,mmas,eas,ras,acs) localsearch "--localsearch " c (0, 1, 2, 3) alpha "--alpha " r (0.00, 5.00) beta "--beta " r (0.00, 10.00) rho "--rho " r (0.01, 1.00) ants "--ants " i (5, 100) nnls "--nnls " i (5, 50) | localsearch %in% c(1, 2, 3) q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) rasrank "--rasranks " i (1, "ants") | algorithm == "ras" elitistants "--elitistants " i (1, 750) | algorithm == "eas" [forbidden] ## Examples of valid logical operators are: ## == != >= <= > < & | ! %in% (alpha == 0.0) & (beta == 0.0) \end{CodeInput} \caption{Parameter file (\code{parameters.txt}) for tuning \ACOTSP.}\label{fig:acotsp_parameters} \end{figure} \subsubsection{Parameters R format}\label{sec:parameters object} The target parameters are stored in an \aR~\code{list} that you can obtain from the \aR console using the following command: <<readParameters,prompt=FALSE, eval=FALSE>>= parameters <- readParameters(file = "parameters.txt") @ See the help of the \code{readParameters} function (\code{?readParameters}) for more information. The structure of the parameter list that is created is as follows: \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{0.98\linewidth}{@{}rX} \code{names} & Vector that contains the names of the parameters. \\ \code{types} & Vector that contains the type of each parameter 'i', 'c', 'r', 'o'. \\ \code{switches} & Vector that contains the labels of the parameters. e.g., switches to be used for the parameters on the command line. \\ \code{domain} & List of vectors, where each vector may contain two values (minimum, maximum) for real and integer parameters, or a set of values for categorical and ordinal parameters.\\ \code{conditions} & List of \aR logical expressions, with variables corresponding to parameter names.\\ \code{isFixed} & Logical vector that specifies which parameter is fixed and, thus, it does not need to be tuned.\\ \code{transform} & Vector that contains the transformation of each parameter. Currently, it can take values \code{``''} (no transformation, default) of \code{``log''} (natural logarithmic transformation).\\ \code{nbParameters} & An integer, the total number of parameters.\\ \code{nbFixed} & An integer, the number of parameters with a fixed value.\\ \code{nbVariable} & Number of variable (i.e., to be tuned) parameters.\\ \code{isDependent} & Logical vector that specifies which parameter defines a dependent domain.\\ \code{forbidden} & List of R logical expressions that cannot evaluate to \code{TRUE} for any evaluated configuration.\\ \end{tabularx} \end{center} The following example shows the structure of the \code{parameters} \aR object for the \code{algorithm}, \code{ants} and \code{q0} parameters of the \ACOTSP scenario: <<parametersetup,eval=TRUE,include=FALSE>>= # Setup example parameters <- readParameters(text=' algorithm "--" c (as,mmas,eas,ras,acs) ants "--ants " i (5, 100) q0 "--q0 " r (0.0, 1.0) | algorithm %in% c("acs") ') @ <<parameterlist,eval=TRUE, prompt=TRUE, size='normalsize', comment="">>= str(parameters, vec.len = 10) @ \subsection{Target algorithm runner} \label{sec:runner} The evaluation of a candidate configuration on a single instance is done by means of a user-given auxiliary program or, alternatively, a user-given \aR function. The function (or program name) is specified by the option \parameter{targetRunner}. The \parameter{targetRunner} must return the cost value (\eg cost of the best solution found) of the evaluation; unless computing the cost requires information from all the configurations evaluated on an instance, \eg when evaluating multi-objective algorithms with unknown normalisation bounds (see \autoref{sec:evaluator} for details). \begin{xwarningbox} The objective of \irace is to minimize the cost value returned by the target algorithm. If you wish to maximize, you can multiply the cost by \code{-1} before returning it to \irace. \end{xwarningbox} \subsubsection{Target runner as an executable program} When \parameter{targetRunner} is an auxiliary executable program, it is invoked for each candidate configuration, passing as arguments: % \begin{center} \small\code{<id_configuration>\ <id_instance>\ <seed>\ <instance>\ [bound]\ <configuration>} \end{center} % \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{1\linewidth}{@{}rX} \code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\ \code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\ \code{seed} & seed for the random number generator to be used for this evaluation, ignore the seed for deterministic algorithms;\\ \code{instance} & string giving the instance to be used for this evaluation;\\ \code{bound} & optional execution time bound. Only provided when the \parameter{boundMax} option is set in the scenario, see \autoref{sec:capping};\\ \code{configuration} & the pairs parameter label-value that describe this candidate configuration. Typically given as command-line switches to be passed to the executable program.\\ \end{tabularx} \end{center} The experiment list shown in \autoref{sec:runner fx}, would result in the following execution line: %<<target_runner_ex1, engine='bash', eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] target-runner 1 113 734718556 /home/user/instances/tsp/2000-533.tsp \ --eas --localsearch 0 --alpha 2.92 --beta 3.06 --rho 0.6 --ants 80 \end{lstlisting} %@ The command line switches that describe the candidate configuration are constructed by appending to each parameter label (switch), \emph{without separator}, the value of the parameter, following the order given in the parameter table. The program \parameter{targetRunner} must print a real number, which corresponds to the cost measure of the candidate configuration for the given instance and optionally its execution time (mandatory when \parameter{maxTime} is used and/or when the \parameter{capping} option is enabled). The working directory of \parameter{targetRunner} is set to the execution directory specified by the option \parameter{execDir}. This allows the user to execute independent runs of \irace in parallel using different values for \parameter{execDir}, without the runs interfering with each other. \subsubsection{Target runner as an R function} \label{sec:runner fx} When \parameter{targetRunner} is an \aR function, it is invoked for each candidate configuration as: % <<targetRunner,prompt=FALSE, eval=FALSE>>= targetRunner(experiment, scenario) @ % \noindent where \code{experiment} is a list that contains information about configuration and instance to execute one experiment, and \code{scenario} is the scenario list. The structure of the \code{experiment} list is as follows: \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{1\linewidth}{@{}rX} \code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\ \code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\ \code{seed} & seed to be used for this evaluation;\\ \code{instance} & string giving the instance to be used for this evaluation;\\ \code{bound} & optional execution time bound;\\ \code{configuration} & 1-row data frame with a column per parameter name;\\ \end{tabularx} \end{center} The following is an example of an experiment list for the \ACOTSP scenario: <<experimentlist,eval=TRUE,size='normalsize', prompt=TRUE, comment="">>= print(experiment) @ If you need the \code{switches} of each parameter (or any other information about the parameters), you can obtain it from \code{scenario$parameters}. You can find an example that calls MATLAB from \aR using this approach here: \url{https://github.com/MLopez-Ibanez/irace/blob/master/inst/examples/matlab/scenario.txt}. If \parameter{targetEvaluator} is \code{NULL}, then the \parameter{targetRunner} function must return a list with at least one element \code{"cost"}, the numerical value corresponding to the evaluation of the given configuration on the given instance. A cost of \code{Inf} is accepted and results in the immediate rejection of the configuration (see \autoref{sec:reject}). If the scenario option \parameter{maxTime} is non-zero or if the \parameter{capping} option is enabled, then the list must contain at least another element \code{"time"} that reports the execution time for this call to \code{targetRunner}. The return list may also contain the following optional elements that are used by \irace for reporting errors in \code{targetRunner}: % \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{1\linewidth}{@{}rX} \code{error} & is a string used to report an error;\\ \code{outputRaw} & is a string used to report the raw output of calls to an external program or function;\\ \code{call} & is a string used to report how \code{targetRunner} called an external program or function;\\ \end{tabularx} \end{center} \subsection{Target evaluator} \label{sec:evaluator} Normally, \parameter{targetRunner} returns the cost of the execution of a candidate configuration (see \autoref{sec:runner}). However, there are cases when the cost evaluation must be delayed until all candidate configurations in a race have been executed on a instance. The \parameter{targetEvaluator} option defines an auxiliary program (or an \aR function) that allows postponing the evaluations of the candidate configurations. For each instance seen, the program \parameter{targetEvaluator} is only invoked after all the calls to \parameter{targetRunner} for all alive candidate configurations on the same instance have already finished. \begin{xwarningbox} When using \parameter{targetEvaluator}, \parameter{targetRunner} must not return the evaluation of the configuration. If \parameter{maxTime} is used, \parameter{targetRunner} must return only execution time. \end{xwarningbox} As an example, \parameter{targetEvaluator} may be used to dynamically find normalization bounds for the output returned by an algorithm for each individual instance. In this case, \parameter{targetRunner} will save the output of the algorithm, then the first call to \parameter{targetEvaluator} will examine the output produced by all calls to \parameter{targetRunner} for the same instance, update the normalization bounds and return the normalized output. Subsequent calls to \parameter{targetEvaluator} for the same instance will simply return the normalized output. A similar need arises when using quality measures for multi-objective optimization algorithms, such as the hypervolume, which typically require specifying reference points or sets. By using \parameter{targetEvaluator}, it is possible to dynamically compute the reference points or sets while \irace is running. Examples are provided at \path{examples/hypervolume}. See also \autoref{sec:multi objective} for more information on how to tune multi-objective algorithms. \subsubsection{Target evaluator executable program} When \parameter{targetEvaluator} is an auxiliary executable program, it is invoked for each candidate with the following arguments: % \begin{center} \small\code{<id_configuration>\ <id_instance>\ <seed>\ <instance>\ <num_configurations>\ <all_conf_id>} \end{center} % \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{1\linewidth}{@{}rX} \code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\ \code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\ \code{seed} & seed to be used for this evaluation;\\ \code{instance} & string giving the instance to be used for this evaluation;\\ \code{num_configurations} & number of alive candidate configurations;\\ \code{all_conf_id} & list of IDs of the alive configurations separated by whitespace.\\ \end{tabularx} \end{center} The \parameter{targetEvaluator} executable must print a numerical value corresponding to the cost measure of the candidate configuration on the given instance. \subsubsection{Target evaluator R function} When \parameter{targetEvaluator} is an \aR function, it is invoked for each candidate configuration as:% % <<targetEvaluator, eval=FALSE>>= targetEvaluator(experiment, num_configurations, all_conf_id, scenario, target_runner_call) @ % \noindent where \code{experiment} is a list that contains information about one experiment (see \autoref{sec:runner fx}), \code{num_configurations} is the number of configurations alive in the race, \code{all_conf_id} is the vector of IDs of the alive configurations, \code{scenario} is the scenario list and \code{target_runner_call} is the string of the \parameter{targetRunner} execution line. The function \parameter{targetEvaluator} must return a list with one element \code{"cost"}, the numerical value corresponding to the cost measure of the given configuration on the given instance. The return list may also contain the following optional elements that are used by \irace for reporting errors in \code{targetEvaluator}: % \begin{center} \renewcommand{\arraystretch}{1.2} \begin{tabularx}{1\linewidth}{@{}rX} \code{error} & is a string used to report an error;\\ \code{outputRaw} & is a string used to report the raw output of calls to an external program or function;\\ \code{call} & is a string used to report how \code{targetEvaluator} called an external program or function;\\ \end{tabularx} \end{center} \subsection{Training instances}\label{sec:training} The \irace options \parameter{trainInstancesDir} and \parameter{trainInstancesFile} specify where to find the training instances. % If you only set the value of \parameter{trainInstancesDir} (e.g., to \path{./Instances}), \irace will consider all files within that directory as training instances. Otherwise, the value of \parameter{trainInstancesFile} may specify a text file. The format of this file is one instance per line. Within each line, elements separated by white-space will be parsed as separate arguments to be supplied to \parameter{targetRunner}. This allows defining instance-specific parameter settings. Quoted strings will be parsed as a single argument. % The following example shows a training instance file for the \ACOTSP scenario: \begin{figure}[!ht] \centering% \begin{minipage}{0.4\textwidth} \begin{CodeInput} # Example training instances file 100/100-1_100-2.tsp --time 1 100/100-1_100-3.tsp --time 2 100/100-1_100-4.tsp --time 3 \end{CodeInput} \end{minipage} \caption{Training instances file for tuning \ACOTSP.}\label{fig:acotsp_training} \end{figure} The value of \parameter{trainInstancesDir}, if set, is always prefixed to the instance name, that is, the instances names are treated as relative to this directory. % For example, given the above file as \parameter{trainInstancesFile} and \parameter{trainInstancesDir}\code{="./Instances"}, then a possible invocation of \parameter{targetRunner} would be: % %<<target_runner_cmd, engine='bash', eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] target-runner 1 4 5718 ./Instances/100/100-1_100-2.tsp --time 1 --alpha 2.92 ... \end{lstlisting} %@ Training instances do not need to be files, \irace just passes the elements of each line as arguments to \parameter{targetRunner}, thus each line may denote the name of a benchmark function or a label, plus instance-specific settings, that the target algorithm understands. Each line may even be the command-line parameters required to call an instance generator within \parameter{targetRunner}. When the instances do not represent actual files, then \parameter{trainInstancesDir} is usually set to the empty string (\code{--train-instances-dir=""}). For example, \begin{center} \begin{minipage}{0.4\linewidth} \begin{CodeInput} # Example training instances file rosenbrock_20 --function=12 --nvar 20 rosenbrock_30 --function=12 --nvar 30 rastrigin_20 --function=15 --nvar 20 rastrigin_30 --function=15 --nvar 30 \end{CodeInput} \end{minipage} \end{center} %\caption{Example of training instances file where instances do not correspond to files.}\label{fig:nofile_training} %\end{figure} Optionally, when executing \irace from the \aR console, the list of instances might be provided explicitly by means of the variable \code{scenario$instances}. Thus, the previous example would be equivalent to: <<instance1, prompt=FALSE, eval=FALSE>>= scenario$instances <- c("rosenbrock_20 --function=12 --nvar 20", "rosenbrock_40 --function=12 --nvar 30", "rastrigin_20 --function=15 --nvar 20", "rastrigin_40 --function=15 --nvar 30") @ By default, \irace assumes that the target algorithm is stochastic (the value of the option \parameter{deterministic} is $0$), thus, the same configuration can be executed more than once on the same instance and obtain different results. In this case, \irace generates pairs \code{(instance, seed)} by generating a random seed for each instance. In other words, configurations evaluated on the same instance use the same random seed. This is a well-known variance reduction technique called \emph{common random numbers}~\citep{McG1992vrt}. If all available pairs are used within a run of \irace, new pairs are generated with different seeds, that is, a configuration evaluated more than once per instance will use different random seeds. If \parameter{deterministic} is set to $1$, then each instance will be used at most once per race. This setting should only be used for target algorithms that do not have a stochastic behavior and, therefore, executing the target algorithm on the same instance several times with different seeds does not make sense. \begin{xwarningbox} If \parameter{deterministic} is active and the number of training instances provided to \irace is less than \parameter{firstTest} (default: 5), no statistical test will be performed on the race. \end{xwarningbox} Finally, \irace randomly re-orders the sequence of instances provided. This random sampling may be disabled by using the option \parameter{sampleInstances} (\code{--sample-instances 0}) if keeping the order provided in the instance file is important. \begin{xwarningbox} We advise to always sample instances to prevent biasing the tuning due to the instance order. See also \autoref{sec:het} \end{xwarningbox} \subsection{Initial configurations} \label{sec:initial} The scenario option \parameter{configurationsFile} allows specifying a text file that contains an initial set of configurations to start the execution of \irace. If the number of initial configurations supplied in the file is less than the number of configurations required by \irace in the first iteration, additional configurations will be sampled uniformly at random. The format of the configurations file is one configuration per line, and one parameter value per column. The first line must give the parameter name corresponding to each column (names must match those given in the parameters file). Each configuration must satisfy the parameter conditions (\code{NA} should be used for those parameters that are not enabled for a given configuration) and not be forbidden by the constraints that define forbidden configurations (\autoref{sec:forbidden}), if any. Figure~\ref{fig:acotsp_default} gives an example file that corresponds to the \ACOTSP scenario. \begin{figure}[!ht] \centering \begin{minipage}{0.7\linewidth} \footnotesize \begin{CodeInput} ## Initial candidate configuration for irace algorithm localsearch alpha beta rho ants nnls dlb q0 rasrank elitistants as 0 1.0 1.0 0.95 10 NA NA 0 NA NA \end{CodeInput} \end{minipage} \caption{Initial configuration file (\code{default.txt}) for tuning \ACOTSP.}\label{fig:acotsp_default} \end{figure} We advise to use this feature when a default configuration of the target algorithm exists or when different sets of good parameter values are known. This will allow \irace to start the search from those parameter values and attempt to improve their performance. \subsection{Repairing configurations}\label{sec:repairconf} In some problems, the parameter values require complex constraints that cannot be implemented by constraints defined in the parameter space (\autoref{sec:forbidden}). % The scenario option \parameter{repairConfiguration} can be set to a user-defined \aR function that takes a single configuration generated by irace and returns a ``\emph{repaired}'' configuration, thus allowing the implementation of any rules necessary to satisfy arbitrary constraints on parameter values. % The \parameter{repairConfiguration} function is called after generating a configuration and before checking for forbidden configurations. The first argument is a 1-row \code{data.frame} with parameter names as the column names and the second argument is the \code{parameters} list (\autoref{sec:parameters object}). An example that makes all real-valued parameters sum up to one would be: % <<repairEx,prompt=FALSE, eval=FALSE>>= repairConfiguration = function(configuration, parameters) { isreal <- names(which(parameters$types[colnames(configuration)] == "r")) # This ignores 'digits' c_real <- unlist(configuration[isreal]) c_real <- c_real / sum(c_real) configuration[isreal] <- c_real return(configuration) } @ The following example forces three specific parameters to be in increasing order: % <<repairEx2,prompt=FALSE, eval=FALSE>>= repairConfiguration = function(configuration, parameters) { columns <- c("p1","p2","p3") # cat("Before"); print(configuration) configuration[columns] <- sort(unlist(configuration[columns], use.names=FALSE)) # cat("After"); print(configuration) return(configuration) } @ The above code can be specified directly in the \parameter{scenarioFile}, by default \code{scenario.txt}. %% %% %% %% Parallel processing %% %% %% \section{Parallelization}\label{sec:parallel} A single run of \irace can be done much faster by executing the calls to \parameter{targetRunner} (the runs of the target algorithm) in parallel. There are four ways to parallelize a single run of \irace: \begin{enumerate} \item \textbf{Parallel processes}: The option \parameter{parallel} executes multiple calls to \parameter{targetRunner} in parallel within a single computer, by means of the \pkg{parallel} \aR package. For example, adding \code{--parallel N} to the command line of \irace will launch in parallel up to $N$ calls of the target algorithm. When using this option within a computing cluster, \irace will be submitted as a \emph{job} in some way that tells the cluster to ``reserve'' $N$ CPUs (or tasks depending on the cluster) within a single cluster node (a single machine). \item \textbf{MPI}: By enabling the option \parameter{mpi}, calls to \parameter{targetRunner} will be executed in parallel by using the message passing interface (MPI) protocol (requires the \pkg{Rmpi} \aR package). In this case, the option \parameter{parallel} controls the number of slave nodes used by \irace. For example, adding \code{--mpi 1 --parallel N} to the command-line will create $N$ slaves + 1 master, and execute up to $N$ calls of \parameter{targetRunner} in parallel. The user is responsible for setting up the required MPI environment. MPI is commonly available in computing clusters and requires launching \irace in some particular way. An example script for using MPI mode in a SGE cluster is given at \IRACEHOME{/bin/parallel-irace-mpi}. By default, \irace dynamically balances the load among nodes, however, this may significantly increase communication overhead in some parallel environments and disabling \parameter{loadBalancing} may be faster. \item \textbf{Batch queue mode}: Some computing clusters work by submitting jobs to a batch queue and waiting for the jobs to finish. With the option \parameter{batchmode} (\code{--batchmode} \code{[sge|pbs|} \code{torque|slurm]}), \irace will launch in parallel as many calls of \parameter{targetRunner} as possible and use a cluster-specific method to wait for jobs to finish. In this mode, \parameter{parallel} controls how many jobs are queued and should be set to the queue limit of your cluster. If your cluster type is not supported or not working as expected, please submit a pull request (\url{https://github.com/MLopez-Ibanez/irace/pulls}) adding support to your cluster type. See the examples in \IRACEHOME{/examples/batchmode-cluster/}. \begin{xwarningbox} In batchmode, \irace runs in the submission node of the cluster, hence, \irace is not submitted to the cluster as a job (that is, neither \code{qsub} nor \code{squeue} should be used to invoke \irace itself). The user must call the appropriate job submission command (e.g., \code{qsub} or \code{squeue}) from \parameter{targetRunner} with the appropriate settings for their cluster, that is, \parameter{targetRunner} submits one job to the cluster and prints a single string: The job ID that allows \irace to determine the status of the running job. Moreover, the use of a separate \parameter{targetEvaluator} script is required to collect the results of \parameter{targetRunner} and return them to \irace. \end{xwarningbox} \item \parameter{targetRunnerParallel}: This option allows users to fully control the parallelization of the execution of \parameter{targetRunner}. Its value must be an \aR function that will be invoked by \irace as follows: <<targetRunnerParallel,prompt=FALSE, eval=FALSE>>= targetRunnerParallel(experiments, exec_target_runner, scenario, target_runner) @ % where \code{scenario} is the list describing the configuration scenario (\autoref{sec:scenario}); \code{experiments} is a list that describes the configurations and instances to be executed (see \autoref{sec:runner} for a description); \code{target_runner} is the function that calls the target algorithm and it is the same as \parameter{targetRunner}, if the latter is a function, or it is a call to \code{target_runner_default}, if \parameter{targetRunner} is the path to an executable; and \code{exec_target_runner} is an internal function within \irace that takes care of executing \code{target_runner}, check its output and, possibly, retry in case of error (see \parameter{targetRunnerRetries}). The \parameter{targetRunnerParallel} function should call the given \code{target_runner} function for each element in the \code{experiments} list, possibly using \code{exec_target_runner} as a wrapper. A trivial example would be: <<targetRunnerParallel2,prompt=FALSE, eval=FALSE>>= targetRunnerParallel <- function(experiments, exec_target_runner, scenario, target_runner) { lapply(experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) } @ % However, the user is free to set up the calls in any way, perhaps implementing their own replacement for \code{target_runner} and/or \code{exec_target_runner}. The user may load and call other \aR packages, such as \pkg{batchtools} (\url{https://mllg.github.io/batchtools/}). The only requirement is that the \parameter{targetRunnerParallel} function must return a list of the same length as \code{experiments}, where each element is the output expected from the corresponding call to \parameter{targetRunner} (see \autoref{sec:runner}). The following is an example of the output of a call to \parameter{targetRunnerParallel} with 2 experiments, in which the execution time is not reported: <<targetRunnerParallel3,prompt=FALSE, eval=TRUE>>= print(output) @ % \end{enumerate} The best option will depend on the resources available to you. Option 1 is usually the fastest and simplest to setup. Running on a node (machine) with 128 CPUs will be faster than running on 8 nodes with 16 CPUs because the communication between nodes required by MPI can be very slow depending on the cluster. Option 2 may be faster if irace generates more configurations per iteration than the number of CPUs of a single node. However, depending on the configuration of your cluster, requesting many CPUs may require waiting in the queue a long time. Option 3 may be the slowest since irace has to check the queue frequently. However, irace will start running experiments as soon as 1 CPU is available, thus option 3 may actually finish earlier than the other options if there is always some CPUs available in the cluster but the queue for requesting many CPUs at once is very long. As a rule-of-thumb, if you only have access to a single machine, then you only need option 1. If you have access to a computing cluster with multiple machines, then use option 1 with the maximum number of CPUs that a single node has in your computing cluster. If that number is 64 or more, it should be enough unless a single run of irace evaluates thousands of configurations. Otherwise, investigate option 2. If option 2 does not work, then investigate option 3. %% %% %% %% Testing %% %% %% \section{Testing (Validation) of configurations}\label{sec:testing} Once the tuning process is finished, \irace returns a set of configurations corresponding to the elite configurations at the end of the run, ordered from best to worst. In order to evaluate the generality of these configurations without looking at their performance on the training set, \irace offers the possibility of evaluating these configurations on a test instance set, typically different from the training set used during the tuning phase. These evaluations will use the same settings for parallel execution, \parameter{targetRunner} and \parameter{targetEvaluator}. The test instances can be specified by the options \parameter{testInstancesDir} and/or \parameter{testInstancesFile}, or by setting directly the variable \code{scenario\$testInstances}. These options behave similarly to their counterparts for the training instances (\autoref{sec:training}). In particular, each test instance is assigned a different seed in the same way as done for the training instances. In principle, \irace evaluates each configuration on each testing instance just once, because evaluating one run on $n$ instances is always better than evaluating $n'$ runs on $n/n'$ instances~\citep{IRIDIA-2004-001}. However, if the number of instances is limited, one can always duplicate instances as needed in the \parameter{testInstancesFile}, and \irace will assign a different random seed to each instance. % An example of the output produced by \irace when testing is shown in Fig.~\ref{fig:testing_output}. \afterpage{\clearpage \begin{figure}[p] \RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}% {fontsize=\scriptsize, % frame=none, % top and bottom rule only framesep=0pt, % separation between frame and text rulecolor=\color{Gray}, } % FIXME: This doesn't break long lines, find another alternative. {\ttfamily\scriptsize\ldots \VerbatimInput{irace-acotsp-testing.txt}}% \caption{Sample text output of \irace when evaluating on test instances.}\label{fig:testing_output} \end{figure}} The options \parameter{testNbElites} and \parameter{testIterationElites} control which configurations are evaluated during the testing phase. In particular, setting \code{testIterationElites = 1} will test not only the final set of elite configurations (those returned at the end of the training phase), but also the set of elites at the end of each race (iteration). The option \parameter{testNbElites} limits the maximum number of configurations considered within each set. Some examples: \begin{itemize} \item \code{testIterationElites = 0; testNbElites = 1} means that only the best configuration found during the run of \irace, the final best, will be used in the testing phase. \item \code{testIterationElites = 1; testNbElites = 1} will test, in addition to the final best, the best configuration found at each iteration. \item \code{testIterationElites = 1; testNbElites = 2} will test the two best configurations found at each iteration, in addition to the final best and second-best configurations. \end{itemize} The testing can be also (re-)executed at a later time by using the following \aR command (but you may need to override \code{testNbElites} and \code{testIterationElites}): % <<testing_r, prompt=FALSE, eval=FALSE>>= testing_fromlog(logFile = "./irace.Rdata", testNbElites = 1) @ % The above line will load the scenario setup from \parameter{logFile} to perform the testing. The testing results will be stored in the \aR object \code{iraceResults\$testing}, which is saved in the file specified by \code{scenario\$logFile}. The structure of the object is described in \autoref{sec:output r}. For examples on how to analyse the results see \autoref{sec:analysis}. Another alternative is to test a specific set of configurations using the command-line option \parameter{--only-test} as follows: \begin{lstlisting}[style=BashInputStyle] irace --only-test configurations.txt \end{lstlisting} where \path{configurations.txt} has the same format as the set of initial configurations (\autoref{sec:initial}). %% %% %% %% Recovery %% %% %% \section{Recovering \irace runs}\label{sec:recovery} Problems like power cuts, hardware malfunction or the need to use computational power for other tasks may occur during the execution of \irace, terminating a run before completion. At the end of each iteration, \irace saves an \aR data file (\parameter{logFile}, by default \code{"./irace.Rdata"}) that not only contains information about the tuning progress (\autoref{sec:output r}), but also internal information that allows recovering an incomplete execution. To recover an incomplete \irace run, set the option \parameter{recoveryFile} to the log file previously produced, and \irace will continue the execution from the last saved iteration. The state of the random generator is saved and loaded, therefore, as long as the execution is continued in the same machine, the obtained results will be exactly the same as executing \irace in one step (external factors, such as CPU load and disk caches, may affect the target algorithm and that may affect the results). You can specify the \parameter{recoveryFile} from the command-line or from the scenario file, and execute \irace as described in \autoref{sec:execution}. For example, from the command-line use: \begin{lstlisting}[style=BashInputStyle] irace --recovery-file "./irace-backup.Rdata" \end{lstlisting} \begin{xwarningbox} When recovering a previous run, \irace will try to save data on the file specified by the \parameter{logFile} option. Thus, you must specify different files for \parameter{logFile} and \parameter{recoveryFile}. Before recovering, we strongly advise to rename the saved \aR data file as in the example above, which uses \code{"irace-backup.Rdata"}. \end{xwarningbox} \hide{% %\LESLIE{Actually, there other variables we allow to change...what we shoudl say about this?} % MANUEL: I would say nothing for now until we fix this properly. When recovery is done you can modify some \irace options from the command-line or from the scenario file. \begin{itemize} \item \parameter{execDir} \item \parameter{logFile} \item \parameter{debugLevel} \item \parameter{parallel} \item \parameter{loadBalancing} \item \parameter{mpi} \item \parameter{batchmode} \end{itemize} It is not possible (and in most cases not correct) to change the scenario options, given that the previous results can become invalid. However there are some cases in which this options must be changed, for example, if you have instances that are files, it might be the case that the instances are not available at the same location. To change the location of them you should modify directly the \aR data file, please be careful not to change the order of the instances, because this would make the results obtained by \irace invalid. To change the instances path, open the \aR console and use: <<change_recover, prompt=TRUE, eval=FALSE>>= iraceResults <- read_logfile("./tuning/irace.Rdata") new_path <- "./experiments/tuning/instances/" iraceResults$scenario$instances <- paste0(new_path, basename(iraceResults$scenario$instances)) save(iraceResults, file="./tuning/irace.Rdata") @ This example can also can be applied to the target execution files \parameter{targetRunner} and \parameter{targetEvaluator}. } \begin{xwarningbox} Do not change anything in the log file or the scenario file before recovering, as it may have unexpected effects on the recovered run of \irace. In case of doubt, please contact us first (\autoref{sec:contact}). In particular, it is not possible to continue a run of \irace by recovering with a larger budget. Results will \textbf{not} be the same as running \irace from the start with the largest budget. An alternative is to use the final configurations from one run as the initial configurations of a new run. \end{xwarningbox} \begin{xwarningbox} If your scenario uses \parameter{targetEvaluator} (\autoref{sec:evaluator}) and \parameter{targetEvaluator} requires files created by \parameter{targetRunner}, then recovery will fail if those files are not present in the \parameter{execDir} directory. This can happen, for example, if you recover from a different directory than the one from which irace was initially executed, or when \parameter{execDir} is set to a temporary directory for every irace run. Thus, you need to copy the contents of the previous \parameter{execDir} into the new one. \end{xwarningbox} %% %% %% %% Output and analysis of results %% %% %% \section{Output and results} During its execution, \irace prints information about the progress of the tuning in the standard output. Additionally, after each iteration, an \aR data file is saved (\parameter{logFile} option) containing the state of \irace. \subsection{Text output}\label{sec:output text} Figure~\ref{fig:output} shows the output, up to the end of the first iteration, of a run of elitist \irace applied to the \ACOTSP scenario with 1000 evaluations as budget. \afterpage{\clearpage \begin{figure}[p] \RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}% {fontsize=\scriptsize, % frame=none, % top and bottom rule only framesep=0pt, % separation between frame and text rulecolor=\color{Gray}, firstline=1, lastline=68, } % FIXME: This doesn't break long lines, find another alternative. \VerbatimInput{irace-acotsp-stdout.txt}% \caption{Sample text output of \irace.}\label{fig:output} \end{figure}} First, \irace gives the user a warning informing that it has found a file with the default scenario filename and it will use it. Then, general information about the selected \irace options is printed: \begin{itemize} \item \parameter{nbIterations} indicates the minimum number of iterations \irace has calculated for the scenario. Depending on the development of the tuning the final iterations that are executed can be more. \item \parameter{minNbSurvival} indicates the minimum number of alive configurations that are required to continue a race. When less configurations are alive the race is stopped and a new iteration begins. \item \code{nbParameters} is the number of parameters of the scenario. \item \parameter{seed} is the number that was used to initialize the random number generator in \irace. \item \code{confidence level} is the confidence level of the statistical test. \item \code{budget} is the total number of evaluations available for the tuning. \item \code{time budget} is the maximum execution time available for the tuning. \item \parameter{mu} is a value used for calculating the minimum number of iterations. \item \parameter{deterministic} indicates if the target algorithm is assumed to be deterministic. \end{itemize} At each iteration, information about the progress of the execution is printed as follows: \begin{itemize} \item \code{experimentsUsedSoFar} is the number of experiments from the total budget that have been used up to the current iteration. \item \code{timeUsed} is the execution time used so far in the experiments. Only available when reported in the \parameter{targetRunner} (activate it with the \parameter{maxTime} option). \item \code{remainingBudget} is the number of experiments that have not been used yet. \item \code{timeEstimate} estimation of the mean execution time. This is used to calculate the remaining budget when \parameter{maxTime} is used. \item \code{currentBudget} is the number of evaluations \irace has allocated to the current iteration. \item \parameter{nbConfigurations} is the number of configurations \irace will use in the current iteration. In the first iteration, this number of configurations include the initial configurations provided; in later iterations, it includes the elite configurations from the previous iterations. \end{itemize} After the iteration information, a table shows the progress of the iteration execution. Each row of the table gives information about the execution of an instance in the race. The first column contains a symbol that describes the results of the statistical test: % \begin{description} \item \code{|x|} No statistical test was performed for this instance. The options \parameter{firstTest} and \parameter{eachTest} control on which instances the statistical test is performed. \item \code{|-|} Statistical test performed and configurations have been discarded. The column \code{Alive} gives an indication of how many configurations have been discarded. \item \code{|=|} Statistical test performed and no configurations have been discarded. This means \irace needs to evaluate more instances to identify the best configurations. \item \code{|!|} This indicator exists only for the elitist version of \irace. It indicates that the statistical test was performed and some elite configurations appear to show bad performance and could be discarded but they are kept because of the elitist rules. See option \parameter{elitist} in \autoref{sec:irace options} for more information. \end{description} Other columns have the following meaning: \begin{description} \item \code{Instance}: Index of \code{(instance, seed)} pair executed. This number corresponds to the row in the data frame returned by \code{get_instanceID_seed_pairs()}. See \autoref{sec:output r} for more information. This index is different from the instance ID passed to \parameter{targetRunner}. \item \code{Bound}: Only when \parameter{capping} is enabled. Execution time used as bound for the execution of new candidate configurations. \item \code{Alive}: Number of configurations that have not been discarded after the statistical test was performed. \item \code{Best}: ID of the best configuration according to the instances seen so far in this race (i.e., not including previous iterations). \item \code{Mean best}: Mean cost value of the best configuration across the instances seen so far in this race (not globally). Equivalent to the concept of ``iteration-best'' in other algorithms. \item \code{Exp so far}: Number of experiments performed so far. \item \code{W time}: Wall-clock time spent on this instance. \item \code{rho}, \code{KenW}, and \code{Qvar}: Spearman's rank correlation coefficient rho, Kendall's concordance coefficient W, and a variance measure described in \cite{SchHoo2012quanti}, respectively, of the configurations across the instances evaluated so far in this iteration. These measures evaluate how consistent is the performance of the configurations across the instances. Values close to 1 for \code{rho} and \code{KenW} and values close to 0 for \code{Qvar} indicate that the scenario is highly homogeneous. For heterogeneous scenarios, we provide advice in \autoref{sec:het}. \end{description} Finally, \irace outputs the best configuration found and a list of the elite configurations. The elite configurations are configurations that did not show statistically significant difference during the race; they are ordered according to their mean performance on the executed instances. \subsection[R data file (logFile)]{\aR data file (\code{logFile})}\label{sec:output r} The \aR data file created by \irace (by default as \code{irace.Rdata}, see option \parameter{logFile}) contains an object called \code{iraceResults}. You can load this file in the \aR console with: <<load_rdata, prompt=FALSE, eval=FALSE>>= logfile <- system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE) iraceResults <- read_logfile(logfile) @ The \code{iraceResults} object is a list, and the elements of a list can be accessed in \aR by using the \code{\$} or \code{[[]]} operators: <<show_version, prompt=TRUE, eval=TRUE, comment="">>= iraceResults$irace_version @ \noindent% The \code{iraceResults} list contains the following elements: \begin{itemize} \item \code{scenario}: The scenario \aR object containing the \irace options used for the execution. See \autoref{sec:irace options} and the help of the \pkg{irace} package; open an \aR console and type: \code{?defaultScenario}. See \autoref{sec:irace options} for more information. \item \code{parameters}: The parameters \aR object containing the description of the target algorithm parameters. See \autoref{sec:target parameters}. \item \code{allConfigurations}: The target algorithm configurations generated by \irace. This object is a \code{data frame}, each row is a candidate configuration; the first column (\code{.ID.}) indicates the internal identifier of the configuration; the final column (\code{.PARENT.}) is the identifier of the configuration from which the current configuration was sampled; and the remaining columns correspond to the parameter values; each column is named as the parameter name specified in the parameter object. <<show_configurations, prompt=TRUE, eval=TRUE, comment="">>= head(iraceResults$allConfigurations) @ \item \code{allElites}: A list that contains one element per iteration. Each element contains the internal identifier of the elite candidate configurations of the corresponding iteration (identifiers correspond to \code{allConfigurations$.ID.}). <<show_idelites, prompt=TRUE, eval=TRUE, comment="">>= print(iraceResults$allElites) @ The configurations are ordered by mean performance, that is, the ID of the best configuration corresponds to the first ID. To obtain the values of the parameters of all elite configurations found by \irace use: <<get_elites, prompt=TRUE, eval=TRUE, comment="">>= logfile <- system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE) getFinalElites(logfile, n = 0) @ \item \code{iterationElites}: A vector containing the best candidate configuration ID of each iteration. The best configuration found corresponds to the last one of this vector. <<show_iditelites, prompt=TRUE, eval=TRUE, comment="">>= print(iraceResults$iterationElites) @ One can obtain the full configuration with: <<get_elite, prompt=TRUE, eval=TRUE, comment="">>= last <- length(iraceResults$iterationElites) id <- iraceResults$iterationElites[last] getConfigurationById(iraceResults, ids = id) @ \item \code{rejectedConfigurations}: A vector containing the rejected configurations IDs. These correspond to configurations that produced failed executions and were ignored by \irace during the configuration process. See \autoref{sec:reject} to enable the detection of such configurations. \item \code{experiments}: A matrix with configurations as columns and instances as rows. Column names correspond to the internal identifier of the configuration (\code{allConfigurations$.ID.}). The results of a particular configuration can be obtained using: <<get_experiments, prompt=TRUE, eval=TRUE, comment="">>= # As an example, we use the best configuration found best_config <- getFinalElites(iraceResults, n = 1) best_id <- as.character(best_config$.ID.) # Obtain the results of the best configuration all_exp <- iraceResults$experiments[, best_id] # all_exp is a vector and names(all_exp) is the (instance,seed) index. all_exp # Obtain the results of the first and best configurations all_exp <- iraceResults$experiments[, c("1", best_id)] # all_exp is a matrix: colnames(all_exp) is configurationID and # rownames(all_exp) is the (instance,seed) index. all_exp @ When a configuration was not executed on an instance, its value is \code{NA}. A configuration may not be executed on an instance because: (1) it was not created yet when the instance was used, or (2) it was discarded by the statistical test and not executed on subsequent instances, or (3) the race terminated before this instance was considered. Row names correspond to the row index of the \code{(instance, seed)} pairs in the data frame returned by \code{get_instanceID_seed_pairs()}. The instanceID and seed used for a particular experiment can be obtained with:\sloppy <<get_instance_seed, prompt=TRUE, eval=TRUE, comment="">>= # As an example, we get instanceID, seeds and instances of the experiments # of the best configuration. # We could get the indexes of the instances on which at least one # configuration was executed: pair_index <- which(apply(!is.na(all_exp), 1L, any)) # or the instances on which all configurations were executed: pair_index <- which(apply(!is.na(all_exp), 1L, all)) # but in this example we get the indexes of the instances executed for # the best configuration. pair_index <- which(!is.na(all_exp[, best_id])) instanceID <- get_instanceID_seed_pairs(iraceResults)[["instanceID"]][pair_index] # or get the seeds get_instanceID_seed_pairs(iraceResults)[["seed"]][pair_index] # or obtain the actual instances. iraceResults$scenario$instances[instanceID] # If the instances are of atomic type (integers, floating-point numbers or # character strings), the above is similar to: get_instanceID_seed_pairs(iraceResults, index = pair_index, instances=TRUE) @ \item \code{experimentLog}: A matrix with columns \code{iteration, instance, configuration}. This matrix contains the log of all the experiments that \irace performs during its execution. The \code{instance} column refers to the index of the data frame returned by \code{get_instanceID_seed_pairs()}. When \parameter{capping} is enabled a column \code{bound} is added to log the execution bound applied for each execution. \item \code{softRestart}: A logical vector that indicates if a soft restart was performed on each iteration. If \code{FALSE}, then no soft restart was performed. See option \parameter{softRestart} in \autoref{sec:irace options}. \item \code{state}: A list that contains the state of \irace, the recovery (\autoref{sec:recovery}) is done using the information contained in this object. The probabilistic model of the last elite configurations can be found here by doing: <<get_model, prompt=TRUE, eval=TRUE, comment="">>= # As an example, we get the model probabilities for the # localsearch parameter. iraceResults$state$model["localsearch"] # The order of the probabilities corresponds to: iraceResults$scenario$parameters$domains$localsearch @ The example shows a list that has one element per elite configuration (ID as element name). In this case, \code{localsearch} is a categorical parameter and it has a probability for each of its values. \item \code{testing}: A list that contains the testing results. The list contains the following elements: \begin{itemize} \item \code{experiments}: Matrix of experiments in the same format as the \code{iraceResults$experiments} matrix. The column names indicate the candidate configuration identifier and the row names contain the name of the instances. <<get_test_exp, prompt=TRUE, eval=TRUE, comment="">>= # Get the results of the testing iraceResults$testing$experiments @ \item \code{seeds}: The seeds used for the experiments, each seed corresponds to each instance in the rows of the test \code{experiments} matrix. <<get_test_seeds, prompt=TRUE, eval=TRUE, comment="">>= # Get the seeds used for testing iraceResults$testing$seeds @ In the example, instance \code{1000-1.tsp} is executed with seed \Sexpr{iraceResults$testing$seeds[1]}. \end{itemize} \end{itemize} \subsection{Analysis of results}\label{sec:analysis} The final configurations returned by \irace are the elites of the final race. They are reported in decreasing order of performance, that is, the best configuration is reported first. If testing is performed, you can further analyze the resulting best configurations by performing statistical tests in \aR: % FIXME: iraceplot % or just plotting the results: %<<plot_test, fig.pos="tbp", fig.align="center", fig.height = 4, fig.width = 8, out.width='0.85\\textwidth', fig.cap="Boxplot of the testing results of the best configurations.", prompt=TRUE, eval=TRUE, comment="">>= <<wilcox_test,prompt=TRUE, eval=TRUE, comment="">>= results <- iraceResults$testing$experiments # Wilcoxon paired test conf <- gl(ncol(results), # number of configurations nrow(results), # number of instances labels = colnames(results)) pairwise.wilcox.test (as.vector(results), conf, paired = TRUE, p.adj = "bonf") @ % # Plot the results % configurationsBoxplot (results, ylab = "Solution cost") The Kendall concordance coefficient (\code{W}) and the Spearman's rho can be applied over data that has the characteristics of the data obtained in the testing, that is a full matrix where all configurations are executed in all instances. \code{W} can show if the configurations tested have an homogeneous performance on the used instances set. If evidence of an heterogeneous scenario found we recommend to make some adjustments in the \irace options as described in \autoref{sec:het}. <<conc, prompt=TRUE, eval=TRUE, comment="">>= irace:::concordance(iraceResults$testing$experiments) @ % %% FIXME: iraceplot % During the tuning, \irace iteratively updates the sampling models of the % parameters to focus on the best regions of the parameter search space. The % frequency of the sampled configurations can provide insights on the parameter % search space. We provide a function for plotting the % frequency of the sampling of a set of configurations. For more information on this function, please see the \aR help, type in the % \aR console: \code{?parameterFrequency}. The following example % plots the frequency of the parameters sampled during one \irace run: % <<freq, fig.pos="tbp", fig.cap="Parameters sampling frequency.", out.width="\\textwidth",prompt=TRUE, eval=TRUE, comment="">>= % parameterFrequency(iraceResults$allConfigurations, iraceResults$scenario$parameters) % @ %% FIXME: iraceplot % By using parallel coordinates plots, it is possible to analyze how the parameters % interact with each other. For more information on this function, please see the \aR help, type % in the \aR console: (\code{?parallelCoordinatesPlot}). % The following example shows how to create a parallel % coordinate plot of the configurations in the last two iterations of \irace. % <<parcord, fig.pos="tbp", fig.align="center", out.width="0.7\\textwidth", fig.cap="Parallel coordinate plots of the parameters of the configurations in the last two iterations of a run of \\irace.", prompt=FALSE, eval=TRUE, comment="">>= % # Get last iteration number % last <- length(iraceResults$iterationElites) % # Get configurations in the last two iterations % conf <- getConfigurationByIteration(iraceResults, % iterations = c(last - 1, last)) % parallelCoordinatesPlot (conf, iraceResults$scenario$parameters, % param_names = c("algorithm", "alpha", "beta", "rho", % "q0", "rasrank"), % hierarchy = FALSE) % @ %%% FIXME: This plot is completely misleading. We should either calculate the %%% performance over the whole training set, or find a way to plot the %%% performance estimated by irace as a confidence interval to account for %%% different number of instances. Or simply plot a heatmap of instances x %%% best-of-each iteration. % It is also possible to plot the performance evolution of the best-so-far configuration over the number of experiments as follows: % %%<<trainEvo, fig.pos="tbp", fig.align="center", out.width='0.75\\textwidth', fig.cap="Training set performance of the best-so-far configuration over number of experiments.", prompt=FALSE, eval=TRUE, comment="">>= % # Get number of iterations % iters <- unique(iraceResults$experimentLog[, "iteration"]) % # Get number of experiments (runs of target-runner) up to each iteration % fes <- cumsum(table(iraceResults$experimentLog[,"iteration"])) % # Get the mean value of all experiments executed up to each iteration % # for the best configuration of that iteration. % values <- sapply(iters, function(k) { % instances <- as.character( % unique(iraceResults$experimentLog[ % iraceResults$experimentLog[, "iteration"] == k, % "instance"])) % return(mean(iraceResults$experiments[ % instances, % as.character(iraceResults$iterationElites[k])]))}) % plot(fes, values, type="s", xlab = "Number of runs of the target algorithm", % ylab = "Estimated mean value over whole training set") % points(fes, values) % @ % FIXME: We should normalize the ranges per instance so since the performance % on different instances is quite different. It is also possible, as shown in Fig.~\ref{fig:testEvo}, to plot the performance on the test set of the best-so-far configuration over the number of experiments as follows: <<testEvo, fig.pos="tb", fig.align="center", out.width='0.7\\textwidth', fig.cap="Testing set performance of the best-so-far configuration over number of experiments. Label of each point is the configuration ID.", prompt=FALSE, eval=TRUE, comment="">>= # Get summary data from the logfile. irs <- irace_summarise(iraceResults) # Get number of iterations iters <- irs$n_iterations # Get number of experiments (runs of target-runner) up to each iteration fes <- cumsum(table(iraceResults$state$experiment_log[["iteration"]])) # Get the mean value of all experiments executed up to each iteration # for the best configuration of that iteration. elites <- as.character(iraceResults$iterationElites) values <- colMeans(iraceResults$testing$experiments[, elites]) stderr <- function(x) sqrt(var(x)/length(x)) err <- apply(iraceResults$testing$experiments[, elites], 2L, stderr) plot(fes, values, type = "s", xlab = "Number of runs of the target algorithm", ylab = "Mean value over testing set", ylim=c(min(values-err),max(values+err))) points(fes, values, pch=19) arrows(fes, values - err, fes, values + err, length=0.05, angle=90, code=3) text(fes, values, elites, pos = 1) @ The \irace package also provides an implementation of the ablation method~\cite{FawHoos2015ablation}. See \autoref{sec:ablation}. \medskip{} Finally, more advanced visualizations of the behavior of \irace are provided by the ACVIZ software package~\cite{DesRitLopPer2021acviz}, which is available at \url{https://github.com/souzamarcelo/acviz}. See an example in Fig.~\ref{fig:acviz}. \begin{figure}[htb] \centering \includegraphics[width=\textwidth]{fig1u-acotsp-instances} \caption{Visualization produced by ACVIZ~\cite{DesRitLopPer2021acviz}.} \label{fig:acviz} \end{figure} %% %% %% %% Advanced options %% %% %% \section{Advanced topics} \subsection{Tuning budget}\label{sec:budget} Before setting the budget for a run of \irace, please consider the number of parameters that need to be tuned, available processing power and available time. The optimal budget depends on the difficulty of the tuning scenario, the size of the parameter space and the heterogeneity of the instances. Typical values range from $1\,000$ to $100\,000$ runs of the target algorithm, although smaller and larger values are also possible. Currently, \irace does not detect whether the given budget allows generating all possible configurations. In such a case, the use of \emph{iterated} racing is unnecessary: One can simply perform a single race of all configurations (see FAQ in \autoref{faq:allconfs}). \Irace provides two options for setting the total tuning budget (\parameter{maxExperiments} and \parameter{maxTime}). The option \parameter{maxExperiments} limits the number of executions of \parameter{targetRunner} performed by \irace. The option \parameter{maxTime} limits the total time of the \parameter{targetRunner} executions. When this latter option is used, \parameter{targetRunner} must return the evaluation cost together with the execution time (\code{"cost time"}). \begin{xwarningbox} When the goal is to minimize the computation time of an algorithm, and you wish to use \parameter{maxTime} as the tuning budget, \parameter{targetRunner} must return the time also as the evaluation cost, that is, return the time twice as \code{"time time"}. \end{xwarningbox} \begin{xwarningbox} When using \parameter{targetEvaluator} and using \parameter{maxTime} as tuning budget, \parameter{targetRunner} just returns the time (\code{"time"}) and \parameter{targetEvaluator} returns the cost. \end{xwarningbox} When using \parameter{maxTime}, \irace estimates the execution time of each \parameter{targetRunner} execution before the configuration. The amount of budget used for the estimation is set with the option \parameter{budgetEstimation} (default is $2\%$). The obtained estimation is adjusted after each iteration using the obtained results and it is used to estimate the number of experiments that can be executed. Internally, \irace uses the number of remaining experiments to adjust the number of configurations tested in each race. \subsection{Multi-objective tuning}\label{sec:multi objective} Currently, \irace only optimizes one cost value at a time, which can be solution cost, computation time or any other objective that is returned to \irace by the \parameter{targetRunner}. If the target algorithm is multi-objective, it will typically return not a single cost value, but a set of objective vectors (typically, a Pareto front). For tuning such a target algorithm with \irace, there are two alternatives. If the algorithm returns a single vector of objective values, they can be aggregated into one single number by using, for example, a weighted sum. Otherwise, if the target algorithm returns a set of objective vectors, a unary quality metric (\eg~the hypervolume) may be used to evaluate the quality of the set.\footnote{An implementation is publicly available at \url{http://lopez-ibanez.eu/hypervolume}~\cite{FonPaqLop06:hypervolume}} % The first option is simple, it requires to devise a formula that can aggregate % the objectives in a way that balances the importance of all of them. This might % not be an easy task in some scenarios, and therefore using a more adequate % indicator to evaluate the performance of a multi-objective optimizer, such as % the hypervolume, is strongly advised. %For using the hypervolume as evaluation \irace needs to postpone the evaluation of the %configurations in an instance until all the executions have been completed. Once %the executions are finalized, the evaluation starts by obtaining reference points for %each objective from the configuration results. These points are used to define the %Pareto front and the hypervolume of each configuration %is calculated and the tuning process continues normally. The use of aggregation or quality metrics often requires normalizing the different objectives. If normalization bounds are known a priori for each instance, normalized values can be computed by \parameter{targetRunner}. Otherwise, the bounds may be dynamically computed while running \irace, by using \parameter{targetEvaluator}. In this case, \parameter{targetRunner} will save the output of the algorithm, then the first call to \parameter{targetEvaluator} will examine the output produced by all calls to \parameter{targetRunner} for the same instance, update the normalization bounds and return the normalized output. Subsequent calls to \parameter{targetEvaluator} for the same instance will simply return the normalized output. % A similar approach can be used to dynamically compute the reference points or sets often required by unary quality metrics. For more information about defining a \parameter{targetEvaluator}, see \autoref{sec:evaluator}. Examples of tuning a multi-objective target algorithm using the hypervolume can be found in the examples at \IRACEHOME{/examples/hypervolume} and \IRACEHOME{/examples/moaco}. \subsection{Tuning for minimizing computation time} \label{sec:capping} When using \irace for tuning algorithms that only report computation time to reach a target, \parameter{targetRunner} should return the execution time of a configuration instead of solution cost. When using \parameter{maxTime} as the budget, this means that \parameter{targetRunner} must return twice the execution time since the first value is the minimization objective and the second value is used to track the budget consumed. Starting from version $3.0$, \irace includes an elitist racing procedure that implements an \textbf{adaptive capping mechanism} \citep{PerLopHooStu2017:lion}. Adaptive capping \citep{HutHooLeyStu2009jair} is a configuration technique that avoids the execution of long runs of the target algorithm, focusing the configuration budget in the evaluation of the best configurations found. This is done by bounding the execution time of each configuration based on the best performing candidate configurations. To use adaptive capping, the \parameter{capping} option must be enabled and the \parameter{elitist} irace option must be selected. When evaluating candidate configurations on an instance, \irace calculates an execution bound based on the execution times of the elite configurations. The \parameter{boundType} option defines how the performance of the elite configurations is defined to obtain the execution bound. The default value of \parameter{boundType} calculates the performance ($p_i^s$) of each elite configuration ($s$) as the mean execution time of the instances already executed in the race and the currently executed instance ($i$). The \parameter{cappingType} option specifies the measure used to obtain the elite configurations bound. By default, the execution bound is calculated as the median of the execution times of the elite configurations: % \begin{equation} b_i= \text{Median}_{\theta_s \in \Celite} \{ p_i^s\} \end{equation} The execution bound for new configurations ($j$) is calculated by multiplying the elite configurations bound by the number of instances ($i$) in the execution list and subtracting the mean execution time of the instances executed by the candidate: % \begin{equation} k_i^{'j} = b_{i} \cdot i + \bmin - p^{j}_{i-1} \cdot (i-1) \end{equation} A small constant \bmin is added to account for time measurements errors. These settings are also used to apply a dominance elimination criterion together with the statistical test elimination. The domination criterion is defined as: % \begin{equation}\label{eq:dom} b_i + \bmin < p_i^{j} \end{equation} When elite configurations dominate new configurations, these are eliminated from the race. \begin{xwarningbox} The default statistical test when \parameter{capping} is enabled is \code{t-test}. This test is more appropriate to configure algorithms for optimizing runtime (see \autoref{sec:stat test}). \end{xwarningbox} The execution bound is constantly adjusted by \irace based on the best configurations times, nevertheless, a maximum execution time (\bmax) is never exceeded. This maximum execution time must be defined in the configuration scenario when \parameter{capping} is enabled. To specify the maximum execution bound for the target runner executions use the \parameter{boundMax} option. The final execution bound ($k_i^j$) is calculated by: % \begin{equation}\label{eq:cap} k_i^j = \begin{cases} \bmax &\text{if $k_i^{'j} > \bmax$,} \\ \min \{b_i, \bmax\}& \text{if $k_i^{'j} \leq 0$,}\\ k_i^{'j}&\text{otherwise;} \end{cases} \end{equation} Additionally, the \parameter{boundDigits} option defines the precision of the time bound provided by \irace, the default setting is 0. Timed out executions occur when the maximum execution bound (\parameter{boundMax}) is reached and the algorithm has not achieved successful termination or a defined quality goal. In this case, it is a common practice to apply a penalty known as PARX, in which timeouts are penalized by multiplying \parameter{boundMax} by a constant $X$. The constant $X$ may be set using the \parameter{boundPar} option. Bounded executions are executions that do not achieve successful termination or a defined quality goal in the execution bound ($k_i^j$) set by \irace, which is smaller than \parameter{boundMax}. The \parameter{boundAsTimeout} option replaces the evaluation of bounded executions by the \parameter{boundMax} value. More details about the implementation of adaptive capping can be found in \citet{PerLopHooStu2017:lion}. \begin{xwarningbox} Note that bounded executions are not timed out executions and thus, they will not be penalized by PARX. \end{xwarningbox} \begin{xwarningbox} Penalized evaluations of timed out and bounded executions are only used for the elimination tests and the comparison between the quality of configurations. To calculate execution bounds and computation budget consumed, \irace uses only unpenalized execution times. The unpenalized execution time must be provided by the target runner or target evaluator as described in \autoref{sec:runner} and \autoref{sec:evaluator} . \end{xwarningbox} \begin{xwarningbox} More advanced capping methods that are applicable to minimizing solution cost are available when combining irace with the \texttt{capopt} package described by \citet{SouRitLop2021cap}. \end{xwarningbox} \subsection{Hyper-parameter optimization of machine learning methods} The \irace package can also be used for model selection and hyper-parameter optimization of machine learning (ML) methods. We will next explain a possible setup for one given dataset and using $10$-fold cross-validation (CV). Generalizing to multiple datasets and different resampling strategies, e.g. leave-one-out, is straightforward. First, split the dataset into training, to be used by \irace, and testing, to be used for evaluating the performance of the configuration returned by \irace. A typical split could be $70\%$ and $30\%$, respectively. The training set is used by \irace to perform $10$-fold CV, that is, the data is split into $10$ folds. A single run of the \parameter{targetRunner} will use $9$ folds for training and the remaining fold for validation. Splitting the data into folds can be done at each call of \parameter{targetRunner} or before running \irace, however, it is important that the split is always the same for every call of the \parameter{targetRunner}, i.e., the content of the folds does not change, only which folds are used for training and validation will change. The setup of \irace should be as follows: % \begin{itemize} \item \parameter{trainInstancesFile}\code{="train-instances.txt"}, where this file contains one number per line from $1$ to $10$. This number will tell the \parameter{targetRunner} which fold should be used for validation. \item \parameter{trainInstancesDir}\code{=""}, because the folds are the ``instances'' and you do not have actual instance files. If you want to pass the name of the dataset to the \parameter{targetRunner}, you can specify it either at each line of \code{"train-instances.txt"}, directly in the \parameter{targetRunner}, or as a fixed parameter in the \parameter{parameterFile}. \item \parameter{deterministic}\code{=1} unless it really makes sense to train more than once the same ML model on the same data. If it makes sense, then your \parameter{targetRunner} should use the seed passed by \irace to seed the ML model before training. \item \parameter{sampleInstances}\code{=0} because the folds should already be generated by randomly sampling the dataset. \item \parameter{testType}\code{="t-test} because the performance metrics in ML are typically the mean of the CV results, which assumes that the performance are close to normally distributed. \item \parameter{firstTest}\code{=2} because \irace should discard configurations very aggressively looking for maximum generality. \end{itemize} \noindent% Finally, your \parameter{targetRunner} needs to be able to do the following: \begin{itemize} \item Receive from \irace the hyper-parameter settings, the dataset name and a fold number (the ``instance''). Let us use fold 3 as an example. \item Train the ML model on the whole training set minus fold 3, then validate (score) the model on fold 3 and return the score to \irace (negated if the score must be maximized, because \irace assumes minimization). Since each fold is different, each instance should give a different result. Each row in the table printed by \irace should print something different; otherwise, something is wrong in your setup. \end{itemize} The above is actually 10 times faster than doing 10-fold CV for each call to \parameter{targetRunner}, thus, you should assign to \irace 10 times the budget than what would be assigned to other methods that do a complete 10-fold CV at each step. %% FIXME: They removed the tutorial %% For such a task, we recommend the \pkg{mlr} package~\citep{R:mlr}. %% The following webpage documents how to use \irace for this purpose: \url{https://mlr-org.github.io/mlr-tutorial/devel/html/advanced_tune/index.html} \subsection{Heterogeneous scenarios} \label{sec:het} We classify a scenario as homogeneous when the target algorithm has a consistent performance regarding the instances; roughly speaking, good configurations tend to perform well and bad configurations tend to perform poorly on all instances of the problem. By contrast, in heterogeneous scenarios, the target algorithm has an inconsistent performance on different instances, that is, some configurations perform well for a subset of the instances, while they perform poorly for a different subset. When facing a heterogeneous scenario, the first question should be whether the objective of tuning is to find configurations that perform reasonably well over all instances, even if that configuration is not the best ones in any particular instance (a generalist). If this is not the goal, then it would be better to partition instances into more similar subsets and execute \irace separately on each subset. This will lead to a portfolio of algorithm configurations, one for each subset, and algorithm selection techniques can be used to select the best configuration from the portfolio when facing a new instance. To make sure \irace is not misled by results on few instances, it may be useful to increase the number of instances executed before doing a statistical test using the option \parameter{firstTest}, \eg \code{--first-test 10} (default value is 5), in order to see more instances before discarding configurations. The option \parameter{elitistNewInstances} in elitist \irace (option \parameter{elitist}) can be used to increase the number of new instances executed in each iteration, \eg \code{--elitist-new-instances 5} (default value is 1). If finding an overall good configuration for all the instances is the objective, then we recommend that instances are randomly sampled (option \parameter{sampleInstances}), unless one can provide the instances in a particular order that does not bias the tuning towards any subset. If instances are easily categorized in different classes, then we recommend to create ``blocks'' of instances in \parameter{trainInstancesFile}, where each block should contain one instance from each class. Then set the option \parameter{blockSize} to the number of classes within each block, so that \irace will always see a complete block of instances before eliminating configurations. The value of \parameter{blockSize} will multiply the values of \parameter{firstTest}, \parameter{eachTest} and \parameter{elitistNewInstances}. Randomly sampling instances (\parameter{sampleInstances}\code{=1}) will randomly sample the blocks but not break the blocks. While executing \irace, the homogeneity of the scenario can be observed by examining the values of Spearman's rank correlation coefficient and Kendall's concordance coefficient in the text output of \irace. See \autoref{sec:output text} for more information. \subsection{Choosing the statistical test} \label{sec:stat test} The statistical test used in \irace identifies statistically bad performing configurations that can be discarded from the race in order to save budget. Different statistical tests use different criteria to compare the cost of the configurations, which has an effect on the tuning results. \Irace provides two types of statistical tests (option \parameter{testType}). Each test has different characteristics that are beneficial for different goals: % \begin{itemize} \item Friedman test (\code{F-test}): This test uses the ranking of the configurations to analyze the differences between their performance. This makes the test suitable for scenarios where the scale of the performance metric is not as important to assess configurations as their relative ranking. This test is also indicated when the distribution of the mean performances deviates greatly from a normal distribution. For example, the ranges of the performance metric on different instances may be completely difference and comparing the performance of different configurations using the mean over multiple instances may be deceiving. We recommend to use the \code{F-test} (default when \parameter{capping} is not enabled) when tuning for solution cost and whenever the best performing algorithm should be among the best in as many instances as possible. \item Student's t-test (\code{t-test}): This test uses the mean performance of the configurations to analyze the differences between the configurations.\footnote{The t-test does not require that the performance values follow a normal distribution, only that the distribution of sample means does. In practice, the t-test is robust despite large deviations from the assumptions.} This makes the test suitable for scenarios where the differences between values obtained for different instances are relevant to assess good configurations. We recommend using t-test, in particular, when the target algorithm is minimizing computation time and, in general, whenever the best configurations should obtain the best average solution cost. \end{itemize} The confidence level of the tests may be adjusted by using the option \parameter{confidence}. Increasing the value of \parameter{confidence} leads to a more strict statistical test. Keep in mind that a stricter test will require more budget to identify which configurations perform worse. A less strict test discards configurations faster by requiring less evidence against them and, therefore, it is more likely to discard good configurations. \subsection{Complex parameter space constraints}\label{sec:complex_domains} Some parameters may have complex dependencies. Ideally, parameters should be defined in the way that is more likely to help the search performed by \irace. For example, when tuning a branch and bound algorithm, one may have the following parameters: \begin{itemize} \item branching (\code{b}) that takes values in \code{\{0,1,2,3\}}, where 0 indicates no branching will be used and the rest are different types of branching. \item stabilization (\code{s}) that takes values in \code{\{0,1,2,3,4,5,6,7,8,9,10\}}, of which for \code{b=0} only \code{\{0,1,2,3,4,5\}} are relevant. \end{itemize} In this case, it is not possible to describe the parameter space by defining only two parameters for \irace. An extra parameter must be introduced as follows: \begin{center} \begin{minipage}{0.8\linewidth} \begin{CodeInput} # name label type range condition b "-b " c (0,1,2,3) s1 "-s " c (0,1,2,3,4,5) | b == "0" s2 "-s " c (0,1,2,3,4,5,6,7,8,9,10) | b != "0" \end{CodeInput} \end{minipage} \end{center} Parameters whose values depend on the value of other parameters may also require using extra parameters or changing the parameters and processing them in \parameter{targetRunner}. For example, given the following parameters: % \begin{itemize} \item Population size (\code{p}) takes the integer values $[1,100]$. \item Selection size (\code{s}) takes the same values but no more than the population size, that is $[1,$\code{p}$]$. \end{itemize} In this case, it is possible to describe the parameters \code{p} and \code{s} using surrogate parameters for \irace that represent a ratio of the original interval as follows: \begin{center} \begin{minipage}{0.8\linewidth} \begin{CodeInput} # name label type range p "-p " i (1,100) s_f "-s " r (0.0,1.0) \end{CodeInput} \end{minipage} \end{center} % and \parameter{targetRunner} must calculate the actual value of \code{s} as $\min(\max(\text{round}(\text{\code{s\_f}}\cdot \text{\code{p}}, 1)), 100)$. For example, if the parameter \code{p} has value 50 and the surrogate parameter \code{s\_f} has value 0.3, then \code{s} will have value 15. The processing within \parameter{targetRunner} can also split and join parameters. For example, assume the following parameters: \begin{center} \begin{minipage}{0.8\linewidth} \begin{CodeInput} # name label type range m "-m " i (1,250) e "-e " r (0.0,2.0) \end{CodeInput} \end{minipage} \end{center} These parameters could be used to define a value $\texttt{m}\cdot 10^\texttt{e}$ for another parameter (\code{--strength}) not known by \irace. Then, \parameter{targetRunner} takes care of parsing \code{-m} and \code{-e}, computing the strength value and passing the parameter \code{--strength} together with its value to the target algorithm. More complex parameter space constraints may be implemented by means of the \parameter{repairConfiguration} function (\autoref{sec:repairconf}). \subsection{Unreliable target algorithms and immediate rejection}\label{sec:reject} There are some situations in which the target algorithm may fail to execute correctly. By default, \irace stops as soon as a call to \parameter{targetRunner} or \parameter{targetEvaluator} fails, which helps to detect bugs in the target algorithm. Sometimes the failure cannot be fixed because it is due to system problems, network issues, memory limits, bugs for which no fix is available, or fixing them is impossible because there is no access to the source code. In those cases, if the failure is caused by random errors or transient system problems, one may wish to ignore the error and try again the same call in the hope that it succeeds. The option \parameter{targetRunnerRetries} indicates the number of times a \parameter{targetRunner} execution is repeated if it fails. Use this option only if you know additional repetitions could be successful. If the target algorithm consistently fails for a particular set of configurations, these configurations may be declared as forbidden (\autoref{sec:forbidden}) so that \irace avoids them. On the other hand, if the configurations that cause the problem are unknown, the \parameter{targetRunner} should return \code{Inf} so that \irace immediately rejects the failing configuration. This immediate rejection should be used with care according to the goals of the tuning. For example, a configuration that crashes on a particular instance, \eg by running out of memory, might still be considered acceptable if it gives very good results on other instances. The configurations which were rejected during the execution of \irace are saved in the Rdata output file (see \autoref{sec:output r}). \begin{xwarningbox} If the configuration budget is specified in total execution time (\parameter{maxTime} option), immediate rejected executions must provide the cost and time (which must be \code{Inf 0}). Nevertheless, rejected configurations will be excluded from the execution time estimation and the execution bound calculation. \end{xwarningbox} \subsection{Ablation Analysis}\label{sec:ablation} The ablation method~\cite{FawHoos2015ablation} takes two configurations (source and target) and generates a sequence of configurations that differ between each other just in one parameter, where parameter values in source are replaced by values from target. The sequence can be seen as a ``path'' from the source to the target configuration. This can be used to find new better ``intermediate'' configurations or to analyse the impact of the parameters in the performance. To perform ablation, you can use the \code{ablation()} \aR function % <<ablation, prompt=FALSE, eval=FALSE>>= ablog <- ablation("irace.Rdata", src = 1) plotAblation(ablog) @ % \noindent or the \path{ablation} command-line executable (see more details below). % %<<linux_irace_help,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] ablation -l irace.Rdata --src 1 --plot plot-ablation.pdf --plot-type rank,boxplot \end{lstlisting} %@ You may specify the IDs of the source and target configurations. By default, the source is taken as the first configuration evaluated by \irace and the target as the best overall configuration found. Use the function \code{plotAblation} to visualize the ablation results (Fig.~\ref{fig:testAb}). <<testAb, fig.pos="htb!", fig.align="center", out.width="0.75\\textwidth", fig.cap="Example of plot generated by \\code{plotAblation()}.", prompt=FALSE, eval=TRUE, echo=FALSE>>= logfile <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork=TRUE) plotAblation(logfile) @ The function returns a list containing the following elements: % \begin{description} \item \code{configurations}: A dataframe of configurations tested during ablation. \item \code{instances}: The instances used for the ablation. \item \code{scenario}: Scenario options provided by the user. \item \code{trajectory}: Best configuration IDs at each step of the ablation. \item \code{best}: Best overall configuration found. \end{description} We also provide a command-line executable (\path{ablation.exe} in Windows) that allows you to perform ablation without launching \aR. It is installed in the same location as the \irace command-line executable and has the following options: <<ablation_cmdline, prompt=FALSE, eval=TRUE,echo=FALSE, comment="">>= ablation_cmdline("--help") @ \subsection{Post-selection race}\label{sec:postselection} At the end of the configuration process, \irace will automatically run a post-selection race \citep{YuaStuMonLauBir13} to try to consume any remaining budget. This post-selection race will not generate new configurations. It will only do additional runs on configurations already evaluated in the current run of \irace. First, it will try to evaluate the best configurations found in new instances not seen yet. If this is not possible due to the budget available, then it will evaluate configurations that have been evaluated on fewer instances than the best configuration found. Currently, this automatic post-selection is not supported when the budget is measured in runtime (\parameter{maxTime}. The automatic post-selection can be disabled using the option \parameter{postselection}. It is also possible to manually run a post-selection race after completing a run of \irace using the R function \iracefun{psRace}. <<postsel, prompt=FALSE, eval=FALSE>>= # Execute all elite configurations in the iterations psRace("irace.Rdata", max_experiments = 0.5, iteration_elites=TRUE) # Execute a set of configurations IDs providing budget psRace("irace.Rdata", conf_ids = c(34, 87, 102, 172, 293), max_experiments = 500) @ The execution of the post-selection race adds an element (\code{psrace_log}) to the \code{iraceResults} object saved in the \irace log file (\parameter{logFile}). The post-selection log consists of a list with the following elements: \begin{description} \item\code{configurations}: Configurations used in the post-selection race. \item\code{instances}: Data frame with the instances and seeds used in the post-selection race. \item \code{max_experiments}: Configuration budget assigned to the post-selection race. \item \code{experiments}: Matrix of results generated by the post-selection race, in the same format as the matrix \code{iraceResults$experiments}. Column names are the configuration IDs and row names are the instance IDs. \item \code{elites}: Best configuration(s) found in the post-selection race. \end{description} \subsection[Parameter importance analysis using PyImp]{Parameter importance analysis using \PyImp} The \PyImp\footnote{\url{https://github.com/automl/ParameterImportance}} tool developed by the AutoML group\footnote{\url{https://www.automl.org/}} supports various parameter importance analysis methods using surrogate models. Given a performance dataset of an algorithm configuration scenario, a Random Forest is built to predict performance of all algorithm configurations. Parameter importance analyses are then applied on the prediction model. The model serves as a surrogate for the original target algorithm, so that the algorithm does not need to be executed during the analyses. Three analysis methods are supported, namely fANOVA~\citep{HutHooLey2014icml} (functional analysis of variance), forward selection~\cite{HutHooLey2013lion}, and ablation analysis with surrogates~\cite{BieLinEggFraFawHoo2017}. Note that the \irace package directly supports ablation (without surrogate models) analysis with and without racing (\autoref{sec:ablation}). Although ablation analysis without surrogates may be more time-consuming, results of the surrogate version may be less accurate than the non-surrogate one. The \code{.Rdata} dataset generated by \irace can be used as input for \PyImp. The package \code{irace2pyimp}\footnote{\url{https://github.com/ndangtt/irace2pyimp}} is able to convert an \code{irace.Rdata} file into the input format required by \PyImp. The conversion can be accessed either through the \aR console (function \code{irace2pyimp}), or via command line executable \code{irace2pyimp} (or \code{irace2pyimp.exe} in Windows) provided by the package. You can find the location of the executable by running this command in R: \Sexpr{'system.file(file.path("bin", "irace2pyimp"), package="irace2pyimp", mustWork=TRUE)'} or \Sexpr{'system.file(file.path("bin", "irace2pyimp.exe"), package="irace2pyimp", mustWork=TRUE)'} in Windows. To see the usage of the executable, please run: \code{irace2pyimp --help}. For more information on the \aR function \code{irace2pyimp}, type in the \aR console: \code{?irace2pyimp}. Given as input an \code{irace.Rdata} file, the script will generate the following output files: \begin{itemize} \item \code{params.pcs}: a text file containing the parameter space definition. \item \code{runhistory.json}: a JSON file containing the list of algorithm configurations evaluated during the tuning and the performance data obtained. \item \code{traj_aclib2.json}: a JSON file containing the best configurations after each iteration of \irace. The last configuration will be used as the target configuration in ablation analysis. \item \code{scenario.txt}: a text file containing the definition of the tuning scenario. \item \code{instances.txt}: a text file containing the list of instances. \item \code{features.csv}: a .csv file containing instance features. If no instance features are provided, the index of each instance will be used as a feature. \end{itemize} \PyImp can then be called using the files listed above as input. Several examples on how to use the script and call \PyImp can be found at \code{system.file("/examples/", package="irace2pyimp")}. %\emph{\PyImp installation note:} At the time when this document is written (December 2019), the latest \PyImp release version (1.0.6) has some bugs. %Please check on \url{https://pypi.org/project/PyImp/#history} to see if \PyImp version > 1.0.6 has been released. %If it has, you can easily install \PyImp using \code{pip install pyimp}. %Otherwise, please install \PyImp from source (\url{https://github.com/automl/ParameterImportance}), branch \code{development}. Installation instructions can be found at \url{https://automl.github.io/ParameterImportance/installation.html}. %% %% irace options %% \section{List of command-line and scenario options} \label{sec:irace options} Most \irace options can be specified in the command line using a flag or in the \irace scenario file using the option name (or setting their value in the \code{scenario} list passed to the various \aR functions exported by the package). This section describes the various \irace options that can be specified by the user in this way. \begin{xwarningbox} Relative filesystem paths (e.g., \path{../scenario/}) given in the command-line are relative to the current working directory (the directory at which \irace is invoked). However, paths given in the scenario file are relative to the directory containing the scenario file. See also Table~\ref{tab:paths}. \end{xwarningbox} <<child-demo, child='section/irace-options.Rnw'>>= @ %% %% %% %% FAQ %% %% %% \section{FAQ (Frequently Asked Questions)} \subsection{Is \irace minimizing or maximizing the output of my algorithm?} By default, \irace considers that the value returned by \parameter{targetRunner} (or by \parameter{targetEvaluator}, if used) should be \underline{\textbf{minimized}}. In case of a maximization problem, one can simply multiply the value by -1 before returning it to irace. This is done, for example, when maximizing the hypervolume (see the last lines in \IRACEHOME{/examples/hypervolume/target-evaluator}). \subsection{Are experiments with \irace reproducible?} Short answer: Yes, under some conditions.\\ Long answer: According to the terminology described by \citet{LopBraPaq2021telo}, we define \emph{repeatability} as ``\emph{exactly repeating the original experiment, generating precisely the same results}''. Following this definition, a run of \irace is repeatable under the following conditions: \begin{itemize} \item Same version of \irace. \item Same version of \aR (different versions of \aR may change the behavior of functions used by \irace). \item The behavior of \parameter{targetRunner} is deterministic or exactly reproducible for the same instance, parameter configuration and random seed. Make sure that \parameter{targetRunner} uses the seed provided by \irace to initialize all random number generators used. If the result of \parameter{targetRunner} depends on CPU-time, wall-clock time or system load in any way, then \parameter{targetRunner} is not reproducible and neither will be \irace. \item Same random seed (\parameter{seed}) given to \irace. \item Same scenario options (\autoref{sec:irace options}). Although some options should not affect reproducibility (e.g., \parameter{debugLevel}), maintaining a list of such options will be a huge effort, thus the safest assumption is that any change may break reproducibility. \item Same parameter space (\autoref{sec:target parameters}), including types, domains, conditions and forbidden configurations. The order of the parameters may also affect reproducibility (the name of the parameters should not) because it affects the order in which random numbers are used. \item Same training instances provided and in the same order (\autoref{sec:training}). Even if the instances are sampled randomly (\parameter{sampleInstances}), a different initial order will produce a different sample even with the same random seed. \item Same initial configurations (\autoref{sec:initial}), if any. \end{itemize} \subsection[Is it possible to configure a MATLAB algorithm with irace?] {Is it possible to configure a \MATLAB algorithm with \irace ?} Definitely. There are three main ways to achieve this: \begin{enumerate} \item Edit the \parameter{targetRunner} script to call \MATLAB in a non-interactive way. See the \MATLAB documentation, or the following links.\footnote{\url{http://stackoverflow.com/questions/1518072/suppress-start-message-of-matlab}}\footnote{\url{http://stackoverflow.com/questions/4611195/how-to-call-matlab-from-command-line-and-print-to-stdout-before-exiting}} % You would need to pass the parameter received by \parameter{targetRunner} to your \MATLAB script.\footnote{\url{https://www.mathworks.com/matlabcentral/answers/97204-how-can-i-pass-input-parameters-when-running-matlab-in-batch-mode-in-windows}}\footnote{\url{https://stackoverflow.com/questions/3335505/how-can-i-pass-command-line-arguments-to-a-standalone-matlab-executable-running}} % There is a minimal example in \IRACEHOME{/examples/matlab/}. \item Call \MATLAB code directly from \aR using the \pkg{matlabr} package (\url{https://cran.r-project.org/package=matlabr}). This is a better option if you are experienced in \aR. Define \parameter{targetRunner} as an \aR function instead of a path to a script. The function should call your \MATLAB code with appropriate parameters. \item Another possibility is calling \MATLAB directly from a different programming language and write \parameter{targetRunner} in that programming language, for example, in \proglang{Python} (see examples in \IRACEHOME{/examples/target-runner-python/}).\footnote{\url{https://www.mathworks.com/help/matlab/matlab_external/call-matlab-functions-from-python.html}\\ \url{https://www.mathworks.com/help/matlab/matlab_external/call-user-script-and-function-from-python.html}} \end{enumerate} \subsection{My program works perfectly on its own, but not when running under \irace. Is irace broken?}\label{faq:valgrind} Every time this was reported, it was a difficult-to-reproduce bug, i.e., a \href{https://en.wikipedia.org/wiki/Heisenbug}{Heisenbug}, in the program (target algorithm), not in \irace. % To detect such bugs, we recommend that you use, within \parameter{targetRunner}, a memory debugger (e.g., \href{http://valgrind.org/}{\code{valgrind}}) to run your program. For example, if your program is executed by \parameter{targetRunner} as: % \begin{lstlisting}[style=BashInputStyle] ${EXE} ${FIXED_PARAMS} -i ${INSTANCE} ${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR} \end{lstlisting} % then replace that line with: % \begin{lstlisting}[style=BashInputStyle] valgrind --error-exitcode=1 ${EXE} ${FIXED_PARAMS} -i ${INSTANCE} \ ${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR} \end{lstlisting} If there are bugs in your program, they will appear in \code{\${STDERR}}, thus do not delete those files. Memory debuggers will significantly slowdown your code, so use them only as a means to find what is wrong with your target algorithm. Once you have fixed the bugs, you should remove the use of \code{valgrind}. \subsection{\irace seems to run forever without any progress, is this a bug?}\label{faq:infloop} Every time this problem was reported, the issue was in the target algorithm and not in \irace. Some ideas for debugging this problem: \begin{itemize} \item Check that the target algorithm is really not running nor paused nor sleeping nor waiting for input-output. \item Use \parameter{debugLevel}\code{=3} to see how \irace calls \code{target-runner}, run the same command outside \irace and verify that it terminates. \item Add some output to your algorithm that reports at the very end the runtime and exit code. Verify that this output is printed when \irace calls your algorithm. \item In \code{target-runner}, print something to a log file \emph{after} calling your target algorithm. Verify that this output appears in the log file when \irace is running. \item Set a maximum timeout when calling your target algorithm from \code{target-runner} (see FAQ~\ref{faq:timeout}). \end{itemize} \subsection{My program may be buggy and run into an infinite loop. Is it possible to set a maximum timeout?}\label{faq:timeout} We are not aware of any way to achieve this using \aR. However, in GNU/Linux, it is easy to implement by using the \code{timeout} command\footnote{\url{http://man7.org/linux/man-pages/man1/timeout.1.html}} in \code{targetRunner} when invoking your program. \subsection{When using the mpi option, \irace is aborted with an error message indicating that a function is not defined. How to fix this?}\label{sec:mpi-error} \pkg{Rmpi} does not work the same way when called from within a package and when called from a script or interactively. When \irace creates the slave nodes, the slaves will load a copy of \irace automatically. If the slave nodes are on different machines, they must have \irace installed. If \irace is not installed system-wide, \aR needs to be able to find \irace on the slave nodes. This is usually done by setting \code{R_LIBS}, \code{.libPaths()} or by loading \irace using \code{library()} or \code{require()} with the argument ``\code{lib.loc}''. The settings on the master are not applied to the slave nodes automatically, thus the slave nodes may need their own settings. After spawning the slaves, it is too late to modify those settings, thus modifying the shell variable \code{R_LIBS} seems the only valid way to tell the slaves where to find \irace. If the path is set correctly and the problem persists, please check these instructions: % \begin{enumerate}[leftmargin=*,widest=3] \item Test that \irace and \pkg{Rmpi} work. Run \irace on a single machine (submit node), without calling \code{qsub}, \code{mpirun} or a similar wrapper around \irace or \aR. \item Test loading \irace on the slave nodes. However, jobs submitted by \code{qsub}/\code{mpirun} may load \aR packages using a different mechanism from the way it happens if you log directly into the node (e.g., with \code{ssh}). Thus, you need to write a little \aR program such as: <<faq3, eval=FALSE>>= library(Rmpi) mpi.spawn.Rslaves(nslaves = 10) paths <- mpi.applyLB(1:10, function(x) { library(irace); return(path.package("irace")) }) print(paths) @ Submit this program to the cluster like you would submit \irace (using \code{qsub}, \code{mpirun} or whatever program is used to submit jobs to the cluster). \item In the script \code{bin/parallel-irace-mpi}, the function \code{irace_main()} creates an MPI job for our cluster. You may need to speak with the admin of your cluster and ask them how to best submit a job for MPI. There may be some particular settings that you need. \pkg{Rmpi} normally creates log files; but \irace suppresses those files unless $\texttt{debugLevel} > 0$. \end{enumerate} Please contact us (\autoref{sec:contact}) if you have further problems. \subsection[Error: 4 arguments passed to {.Internal(nchar)} which requires 3]{Error: 4 arguments passed to \code{.Internal(nchar)} which requires 3} This is a bug in \aR 3.2.0 on Windows. The solution is to update your version of \aR. \subsection[Warning: In read.table(filename, header = TRUE, colClasses = "character", : incomplete final line found by \ldots]{Warning: \code{In read.table(filename,} \code{header = TRUE,} \code{colClasses =} \code{"character",} \code{: incomplete final line found by} \ldots} This is a warning given by \aR when the last line of an input file does not finish with the newline character. The warning is harmless and can be ignored. If you want to suppress it, just open the file and press the \code{ENTER} key at the end of the last line of the file to end the final line with a newline. \subsection{How are relative filesystem paths interpreted by \irace?}\label{faq:relpaths} The answer depends on where the path appears. Relative paths may appear as the argument of command-line options, as the value of options given in the scenario file, or within various scripts, functions or instance files. Table \ref{tab:paths} summarizes how paths are translated from relative to absolute. \begin{table}[h!] \centering \caption{Translation of relative to absolute filesystem paths.} \label{tab:paths} \begin{tabular}{rl} \toprule \bf Relative path appears as \ldots & \bf\ldots is relative to \ldots \\ \midrule a string within \parameter{trainInstancesFile} & \parameter{trainInstancesDir}\\ a string within \parameter{testInstancesFile} & \parameter{testInstancesDir}\\ code within \parameter{targetRunner} or \parameter{targetEvaluator} & \parameter{execDir}\\ the value of \parameter{logFile} or \code{--log-file} & \parameter{execDir}\\ the value of other options in the scenario file & the directory containing the scenario file\\ the value of other command-line options & invocation (working) directory of \irace\\ \bottomrule \end{tabular} \end{table} \subsection{My parameter space is small enough that \irace could generate all possible configurations; however, \irace generates repeated configurations and/or does not generate some of them. Is this a bug?}\label{faq:allconfs} Typically, \irace is applied to parameter spaces that are much larger than what can be explored within the budget given. Thus, \irace does not try to detect whether all possible configurations can be evaluated for the given budget and it does not waste computation time to check for repeated configurations. Thus, if the parameter space is actually very small, the initial random sampling performed by \irace may generate repeated configurations and/or never generate some configurations, which is not ideal. If you still want to use (non-iterated) racing, the recommended approach is to provide all configurations explicitly to \irace (\autoref{sec:initial}) and execute a single race (\parameter{nbIterations}\code{=1}) with exactly the number of configurations provided (e.g., \parameter{nbConfigurations}\code{=240}). A future version of \irace may automatically detect this case and switch to non-iterated racing without having to set additional options. Future versions may also implement computationally cheap checks for repeated configurations.\footnote{If you are interested in implementing this, please contact us!} \subsection[On Windows and using target-runner.py (a Python file), I get the error target-runner.py is not executable]{On Windows and using \code{target-runner.py} (a \proglang{Python} file), I get the error ``\code{target-runner.py is not executable}''}\label{faq:py-not-exe} The issue is that \code{.py} files are not executable on their own and you need \code{python.exe} to read the \code{.py} file and execute it. Linux knows how to do this if the first line of the file is ``\code{\#!/usr/bin/python}'', however, Windows doesn't know how to do it. In Windows you have 2 options: \begin{itemize} \item Create a \code{target-runner.bat} file that contains a line similar to (see \path{templates/windows/target-runner.bat}): \begin{lstlisting}[style=BashInputStyle] C:\path\to\python.exe C:\path\to\target-runner.py %instance% %seed% \ %candidate_parameters% 1>%stdout% 2>%stderr% \end{lstlisting} \item Or convert \code{target-runner.py} into an \code{.exe} file, for example, using \code{auto-py-to-exe}\footnote{\url{https://pypi.org/project/auto-py-to-exe/}}, so that you do not need a \code{.bat} file. \end{itemize} \subsection[Error in socketConnection(\ldots) : can not open the connection]{Error in \code{socketConnection("localhost", port = port, server = TRUE,} \code{lock = TRUE,} : can not open the connection} This error may arise if you activate the \parameter{parallel} option of \irace and your \parameter{targetRunner} or \parameter{targetEvaluator} tries to setup a parallel cluster or execute code in parallel in a way that interacts badly with the parallel mechanism in \aR. In this case, you need to either investigate yourself if there is a way for the two parallel mechanisms to co-exist or, if that is not possible, disable parallelism in \irace or in your code. Note that packages or software used by your \parameter{targetRunner} may have a parallel mechanism enabled by default and unknown to you. This is definitely NOT a bug in \irace. \subsection[irace does not print the call to the target-runner with --debug-level 2 when using the --parallel option]{\irace does not print the call to the \parameter{targetRunner} with \parameter{debugLevel}\code{=2} when using the \parameter{parallel}} This is a limitation of Windows or Rstudio. Running without \parameter{parallel} should work. Unfortunately, we cannot fix this limitation in \irace. If you need to understand how \irace calls \parameter{targetRunner} when running in parallel, you can implement a logging mechanism able to handle parallelism directly inside the \parameter{targetRunner}. %% %% %% %% Contact info %% %% %% %FIXME: complete this section %\section{Known problems} \section{Resources and contact information} \label{sec:contact} More information about the package can be found on the \irace webpage: \begin{center} \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/index.html} \end{center} For questions and suggestions please contact the development team through the \irace package Google group: \begin{center} \url{https://groups.google.com/d/forum/irace-package} \end{center} or by sending an email to: \begin{center} \href{mailto:irace-package@googlegroups.com}{irace-package@googlegroups.com} \end{center} \section{Acknowledgements} We would like to thank all the people that directly or indirectly have collaborated in the development and improvement of \irace: % \newlist{acklist}{itemize*}{1} \setlist[acklist]{label={}, before=\unskip{}, afterlabel=\unskip{ }, itemjoin={{,}}, itemjoin*={, and }, after={{.}}} \begin{acklist} \item Prasanna Balaprakash \item Zhi (Eric) Yuan \item Franco Mascia \item Alberto Franzin \item Anthony Antoun \item Esteban Diaz Leiva %MATLAB example \item Federico Caselli % bug reports with fixes \item Pablo Valledor Pellicer % testing of --parallel under Windows \item André de Souza Andrade \item Nguyen Dang (\texttt{nttd@st-andrews.ac.uk}) % implemented \code{irace2pyimp} \end{acklist} \newcommand{\doi}[1]{doi:\hspace{.16667em plus .08333em}\discretionary{}{}{}\href{https://doi.org/#1}{\urlstyle{rm}\nolinkurl{#1}}} \bibliographystyle{abbrvnat} \ifthenelse {\boolean{Release}}{% \bibliography{irace-package}% }{% \bibliography{optbib/abbrev,optbib/journals,optbib/authors,optbib/articles,optbib/biblio,optbib/crossref}% } \newpage \begin{appendices} \section{Installing R} \label{sec:installation} This section gives a quick \aR installation guide that will work in most cases. The official instructions are available at \url{https://cran.r-project.org/doc/manuals/r-release/R-admin.html} \subsection{GNU/Linux} You should install \aR from your package manager. On a Debian/Ubuntu system it will be something like: \begin{lstlisting}[style=BashInputStyle] sudo apt-get install r-base \end{lstlisting} Once \aR is installed, you can launch \aR from the Terminal and from the \aR prompt install the \irace package (see \autoref{sec:irace install}). \subsection{OS X} You can install \aR directly from a CRAN mirror.\footnote{\url{https://cran.r-project.org/bin/macosx/}} % Alternatively, if you use homebrew, you can just brew the \aR formula from the science tap (unfortunately it does not come already bottled so you need to have Xcode\footnote{Xcode download webpage: \url{https://developer.apple.com/xcode/download/}} installed to compile it): %<<R_OS_install,engine='bash',eval=FALSE>>= \begin{lstlisting}[style=BashInputStyle] brew tap homebrew/science brew install r \end{lstlisting} %@ Once \aR is installed, you can launch \aR from the Terminal (or from your Applications), and from the \aR prompt install the \irace package (see \autoref{sec:irace install}). \subsection{Windows} You can install \aR from a CRAN mirror.\footnote{\url{https://cran.r-project.org/bin/windows/}} % We recommend that you install \aR on a filesystem path without spaces, special characters or long names, such as \path{C:\R}. Once \aR is installed, you can launch the \aR console and install the \irace package from it (see \autoref{sec:irace install}). \section{targetRunner troubleshooting checklist} \label{sec:check list} If the \parameter{targetRunner} script fails to return the output expected by \irace, it can be sometimes difficult to diagnose where the problem lies. The more descriptive errors provided by your script, the easier it will be to debug it. If \parameter{targetRunner} enters an infinite loop, irace will wait indefinitely (see FAQ in \autoref{faq:timeout}). If you are using temporary files to redirect the output of your algorithm, check that these files are properly created. We recommend to follow the structure of the example file (\code{target-runner}) provided in \IRACEHOME{/templates}. The following error examples are based on that example file. In case of failure of \parameter{targetRunner}, \irace will print an error on its output describing which execution of \parameter{targetRunner} failed. % Follow this checklist to detect where the problem is: \begin{enumerate}[leftmargin=*,widest=9] \item Make sure that your \code{targetRunner} script or program is at the specified location. If you see this error: \begin{CodeInput} Error: == irace == target runner './tuning/target-runner' does not exist \end{CodeInput} % it means that \irace cannot find the \code{target-runner} file. Check that the file is at the path specified by the error. \item Make sure that your \code{targetRunner} script is an executable file and the user running \irace has permission to execute it. The following errors: % \begin{CodeInput} Error: == irace == target runner './tuning/target-runner' is a directory, not a file \end{CodeInput} or \begin{CodeInput} Error: == irace == target runner './tuning/target-runner' is not executable \end{CodeInput} % mean that your \code{targetRunner} is not an executable file. In the first case, the script is a folder and therefore there must be a problem with the name of the script. In the second case, you must make the file executable, which in GNU/Linux can be done by: % \begin{lstlisting}[style=BashInputStyle] chmod +x ./tuning/target-runner \end{lstlisting} \item If your \code{targetRunner} script calls another program, make sure it is at the location described in the script (variable \code{EXE} in the examples and templates). A typical output for such an error is: % \begin{CodeInput} Error: == irace == running command ''./tuning/target-runner' 1 8 676651103 ./tuning/Instances/1000-16.tsp --ras --localsearch 2 --alpha 4.03 --beta 1.89 --rho 0.02 --ants 37 --nnls 48 --dlb 0 --rasranks 15 2>\&1' had status 1 == irace == The call to target_runner_default was: ./tuning/target-runner 1 8 676651103 ./tuning/Instances/1000-16.tsp --ras --localsearch 2 --alpha 4.03 --beta 1.89 --rho 0.02 --ants 37 --nnls 48 --dlb 0 --rasranks 15 == irace == The output was: Tue May 3 19:00:37 UTC 2016: error: ./bin/acotsp: not found or not executable (pwd: ./tuning/acotsp-arena) \end{CodeInput} % You may test your script by copying the command line shown in the error and executing \code{target-runner} directly on the execution directory (\parameter{execDir}). In this case, the command line is: % \begin{lstlisting}[style=BashInputStyle] ./tuning/target-runner 1 8 676651103 ./tuning/Instances/1000-16.tsp --ras \ --localsearch 2 --alpha 4.03 --beta 1.89 --rho 0.02 --ants 37 --nnls 48 \ --dlb 0 --rasranks 15 \end{lstlisting} This executes the \code{targetRunner} script as \irace does. The output of this script must be only one number. \item If your \code{targetRunner} file is an executable script in Python, \aR, Perl, Bash or some other interpreted programming language, make sure that the interpreter specified in the first line of the file exists at the correct location. For example, if the first line of \code{target-runner.py} is: % \begin{CodeInput} #|/usr/bin/python \end{CodeInput} % Then make sure that \path{/usr/bin/python} exists and it is executable. Otherwise, you will get an error such as: % \begin{CodeInput} Error: == irace == error in running command \end{CodeInput} \item Check that your \code{targetRunner} script is actually returning one number as output. For example: % \begin{CodeInput} Error: == irace == The output of './tuning/target-runner 1 25 365157769 ./tuning/Instances/1000-31.tsp --ras --localsearch 1 --alpha 0.26 --beta 6.95 --rho 0.69 --ants 56 --nnls 10 --dlb 0 --rasranks 7' is not numeric! == irace == The output was: Solution: 24479793 \end{CodeInput} % FIXME: Duplicated above. %% For testing your script, copy the command-line of \code{target-runner} and execute it directly on the execution directory (\parameter{execDir}): %% % %% \begin{lstlisting}[style=BashInputStyle] %% ./tuning/target-runner 1 25 365157769 ./tuning/Instances/1000-31.tsp --ras \ %% --localsearch 1 --alpha 0.26 --beta 6.95 --rho 0.69 --ants 56 \ %% --nnls 10 --dlb 0 --rasranks 7 %% \end{lstlisting} %% This executes the \code{targetRunner} script as \irace does. In the example above, the output of \code{target-runner} is ``\code{Solution: 24479793}'', which is not a number. If \code{target-runner} is parsing the output of the target algorithm, you need to verify that the code only parses the solution cost value. \item Check that your \code{targetRunner} script is creating the output files for your algorithm. If you see an error as: % \begin{CodeInput} == irace == The output was: Tue May 3 19:41:40 UTC 2016: error: c1-9.stdout: No such file or directory \end{CodeInput} The output file of the execution of your algorithm has not been created (check permissions) or has been deleted before the result can be read. \item Other errors can produce the following output: % \begin{CodeInput} == irace == The output was: Tue May 3 19:49:06 UTC 2016: error: c1-23.stdout: Output is not a number \end{CodeInput} % This might be because your \code{targetRunner} script is not executing your algorithm correctly. To further investigate this issue, comment out the line that eliminates the temporary files that saves the output of your algorithm. Similar to this one % \begin{lstlisting}[style=BashInputStyle] rm -f "${STDOUT}" "${STDERR}" \end{lstlisting} % Execute directly the \code{targetRunner} command-line that is provided in the error message, look in your execution directory for the files that are created. Check the \code{.stderr} file for errors and the \code{.stdout} file to see the output that your algorithm produces. \item Some command within \code{targetRunner} may not be working correctly. In that case, you must debug the commands individually exactly as \irace executes them. In order to find where the problem is, print the commands to a log file before executing them. For example: % \begin{lstlisting}[style=BashInputStyle] echo "$EXE ${FIXED_PARAMS} -i $INSTANCE ${CONFIG_PARAMS}" >> ${STDERR}.log $EXE ${FIXED_PARAMS} -i $INSTANCE ${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR} \end{lstlisting} % then look at the \code{${STDERR}.log} file corresponding to the \code{targetRunner} call that failed and execute/debug the last command there. \item If the language of your operating system, the \code{target-runner} or the target algorithm is not English, \irace may not be able to recognize the numbers generated by \code{target-runner}. We recommend that you run \irace, the \code{target-runner} and the target algorithm under an English locale (or make sure that their languages and number format are compatible). \item It is possible that \href{https://en.wikipedia.org/wiki/Heisenbug}{transient bugs} in the target algorithm are only visible when running within \irace, and all commands within \code{targetRunner} appear to work fine when executed directly in the command-line outside \irace. See FAQ in \autoref{faq:valgrind}) for suggestions on how to detect such bugs. \item If your \code{targetRunner} script works when running irace with \parameter{parallel}\code{=0} but it fails when using higher number of cores, this may be due to any number of reasons: \begin{itemize} \item If you submit jobs through a queuing system, the running environment when using the queuing system may not be the same as when you launch \irace yourself. The queuing system may also send the job to different machines depending on the number of CPUs requested. One way to test this is to submit the failing execution of \code{targetRunner} to the queuing system, and specifically to any problematic machine. \item When using MPI, some calls to \code{targetRunner} may run on different computers than the one running the master \irace process. See FAQ in \autoref{sec:mpi-error}. \item Does \code{targetRunner} read or create intermediate files? These files may cause a race condition when two calls to \code{targetRunner} happen at the same time. You have to make sure that parallel runs of \code{targetRunner} do not interfere with each other's files. \item Maybe these files consume too much memory or fill the filesystem when there are simultaneous \code{targetRunner} calls? Moreover, queuing systems have stricter limits for computing nodes than for the submit/host node. \item Does the machine or the queuing system impose any limits on number of processes or CPU/memory/filesystem usage per job? Such limits may only trigger when more than one process is executed in parallel, killing the \code{targetRunner} process before it has a chance to print anything useful. In that case, \irace may not detect the the program finished unexpectedly, only that the expected output was not printed. \end{itemize} \end{enumerate} \section{targetEvaluator troubleshooting checklist} \label{sec:evaluator check list} Even if \parameter{targetRunner} appears to work, the use of \parameter{targetEvaluator} may lead to other problems. The same checklist of \code{targetRunner} can be followed here. In addition, we list here other potential problems unique to \code{targetEvaluator}: \begin{enumerate} \item If \parameter{targetEvaluator} fails only in the second or later iteration, this may because output files or data generated by a previous call to \parameter{targetRunner} are missing. Elite configurations are never re-executed on the same instance and seed pair, that is, \irace will call only once \parameter{targetRunner} for each pair of configuration ID and instance ID. However, \parameter{targetEvaluator} is always re-executed, which takes into account any updated information (normalization bounds, reference sets/points, best-known values, etc.). Thus, any files or data generated by \parameter{targetRunner} for a given configuration must remain available to \parameter{targetEvaluator} as long as that configuration is alive. The list of alive configurations is passed to \parameter{targetEvaluator}, which may decide then which data to keep or remove. \end{enumerate} \section{Glossary} \begin{description} \item[Parameter tuning:] Process of searching good settings for the parameters of an algorithm under a particular tuning scenario (instances, execution time, etc.). \item[Scenario:] Settings that define an instance of the tuning problem. These settings include the algorithm to be tuned (target), budget for the execution of the target algorithm (execution time, evaluations, iterations, etc.), set of problem instances and all the information that is required to perform the tuning. \item[Target algorithm:] Algorithm whose parameters will be tuned. \item[Target parameter:] Parameter of the target algorithm that will be tuned. \item[\irace option:] Configurable option of \irace. \item[Elite configurations:] Best configurations found so far by \irace. New configurations for the next iteration of \irace are sampled from the probabilistic models associated to the elite configurations. All elite configurations are also included in the next iteration. \item[\hypertarget{irace_home}{\path{$IRACE_HOME}:}] The filesystem path where \irace is installed. You can find this information by opening an \aR console and executing: <<R_irace_home2, prompt=FALSE, eval=FALSE>>= system.file(package = "irace") @ \end{description} \section{NEWS} \RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}% {fontsize=\footnotesize, % frame=lines, % top and bottom rule only framesep=1em, % separation between frame and text rulecolor=\color{Gray}, % label=\fbox{\color{Black}NEWS}, labelposition=topline, % % commandchars=\|\(\), % escape character and argument delimiters for % % commands within the verbatim % commentchar=* % comment character } %% NEWS is latin1 but this file is utf8 \begingroup\inputencoding{utf8}% % FIXME: This doesn't break long lines, find another alternative. \VerbatimInput{../NEWS.md}% \endgroup \end{appendices} \end{document} %%% Local Variables: %%% TeX-master: "irace-package.Rnw" %%% End: % LocalWords: iteratively