% !Rnw weave = knitr %%% DO NOT EDIT the .tex file directly since it is generated from the .Rnw %%% sources. %\VignetteEngine{knitr::knitr} %\VignetteIndexEntry{irace package: User Guide} %\VignetteDepends{knitr} %\VignetteCompiler{knitr} \synctex=1 \RequirePackage{xparse} \RequirePackage[dvipsnames]{xcolor} \documentclass[a4paper,english]{article} \usepackage[utf8]{inputenc} \usepackage[T1]{fontenc} \usepackage{lmodern} \usepackage[a4paper]{geometry} % It saves some pages \usepackage[english]{babel} \usepackage{ifthen} \newboolean{Release} \setboolean{Release}{true} \usepackage{calc} \usepackage{afterpage} \usepackage{algorithm,algorithmic} \usepackage{booktabs} \usepackage{tabularx} \usepackage{xspace} \usepackage{amsmath,amssymb} \usepackage{relsize} \usepackage{fancyvrb} \usepackage{underscore} \usepackage{microtype} % \texttt{test -- test} keeps the "--" as "--" (and does not convert it to an en dash) \DisableLigatures{encoding = T1, family = tt* } \usepackage[hyphens]{url} \usepackage{hyperref} \usepackage[numbers]{natbib} \usepackage[nottoc]{tocbibind} %% For autoref \hypersetup{ colorlinks, linkcolor={red!50!black}, citecolor={blue!50!black}, urlcolor={blue!70!black} } \addto\extrasenglish{% \def\sectionautorefname{Section} \let\subsectionautorefname\sectionautorefname \let\subsubsectionautorefname\sectionautorefname } \usepackage[titletoc, title]{appendix} % Fix use with \autoref \newcommand*{\Appendixautorefname}{Appendix} \usepackage{tocloft} \setlength{\cftsubsecnumwidth}{3em}% Set length of number width in ToC for \subsection \usepackage[inline]{enumitem} \setlist[enumerate]{leftmargin=*,widest=00} \setlist[itemize]{leftmargin=1.5em} %% FIXME: listing is very limited, we should use 'minted' \usepackage{listings} \lstdefinestyle{BashInputStyle}{ language=bash,% basicstyle=\ttfamily,% numbers=none,% frame=tb,% rulecolor=\color{lightgray}, % framesep=1ex, framexleftmargin=1ex, columns=fullflexible,% backgroundcolor=\color{yellow!05},% linewidth=\linewidth,% % xleftmargin=1\linewidth,% identifierstyle=\color{darkgray},% keywordstyle=\color{darkgray},% keywordstyle={[2]\color{Cyan}},% keywordstyle={[3]\color{olive}},% stringstyle=\color{MidnightBlue},% commentstyle=\color{RedOrange},% morestring=[b]',% showstringspaces=false } \DefineVerbatimEnvironment{Code}{Verbatim}{} \DefineVerbatimEnvironment{CodeInput}{Verbatim}{fontshape=rm} \DefineVerbatimEnvironment{CodeOutput}{Verbatim}{} \newenvironment{CodeChunk}{}{} \newcommand{\IRACEHOME}[1]{\hyperlink{irace_home}{\path{$IRACE_HOME}}\path{#1}} \providecommand{\keywords}[1]{\textbf{\textit{Index terms---}} #1} % Simple font selection is not good enough. For example, |\texttt{--}| % gives `\texttt{--}', i.e., an endash in typewriter font. Hence, we % need to turn off ligatures, which currently only happens for commands % |\code| and |\samp| and the ones derived from them. Hyphenation is % another issue; it should really be turned off inside |\samp|. And % most importantly, \LaTeX{} special characters are a nightmare. E.g., % one needs |\~{}| to produce a tilde in a file name marked by |\file|. % Perhaps a few years ago, most users would have agreed that this may be % unfortunate but should not be changed to ensure consistency. But with % the advent of the WWW and the need for getting `|~|' and `|#|' into % URLs, commands which only treat the escape and grouping characters % specially have gained acceptance \makeatletter \DeclareRobustCommand\code{\bgroup\@makeother\_\@makeother\~\@makeother\$\@noligs\@codex} \def\@codex#1{\texorpdfstring% {{\text{\normalfont\ttfamily\hyphenchar\font=-1 #1}}}% {#1}\egroup} \makeatother \let\proglang=\textsf \newcommand{\pkg}[1]{{\fontseries{b}\selectfont #1}} \newcommand{\aR}{\proglang{R}\xspace} \newcommand{\MATLAB}{\proglang{MATLAB}\xspace} \newcommand{\eg}{e.g.,\xspace} \newcommand{\SoftwarePackage}{\pkg} \newcommand{\ACOTSP}{\SoftwarePackage{ACOTSP}\xspace} %% How to use this command: % Parameter with one short switch: \defparameter[short]{paramName}{long}{default} % Parameter without short switch: \defparameter{paramName}{long}{default} % Parameter without switch: \defparameter{paramName}{}{default} \newcommand{\defparameter}[4][]{% \item[\code{#2}]\hypertarget{opt:#2}{} ~~ % \ifthenelse{\equal{#3}{}}{}{% \emph{flag:} % \ifthenelse{\equal{#1}{}}{}{% \code{-#1}~~~\emph{or}~~~}% \code{--#3} ~~ }% \emph{default:}~\texttt{#4} \\ } \newcommand{\parameter}[1]{\hyperlink{opt:#1}{\code{#1}}} \newcommand{\iracefun}[1]{\href{https://mlopez-ibanez.github.io/irace/reference/#1.html}{\code{#1()}}} %\usepackage{showlabels} %\showlabels{hypertarget} \newcommand{\irace}{\pkg{irace}\xspace} \newcommand{\Irace}{\pkg{Irace}\xspace} \newcommand{\race}{\pkg{race}\xspace} \newcommand{\FRACE}{\text{F-Race}\xspace} \newcommand{\IFRACE}{\text{I/F-Race}\xspace} \newcommand{\PyImp}{\pkg{PyImp}\xspace} \newcommand{\iraceversion}{\Sexpr{packageVersion("irace")}} \newcommand{\Niter}{\ensuremath{N^\text{iter}}\xspace} \newcommand{\Nparam}{\ensuremath{{N^\text{param}}}\xspace} \newcommand{\iter}{\ensuremath{j}\xspace} \newcommand{\Budget}{\ensuremath{B}\xspace} \newcommand{\Budgetj}{\ensuremath{\Budget_{\iter}}\xspace} \newcommand{\Bused}{\ensuremath{\Budget_\text{used}}\xspace} \newcommand{\Ncand}[1][]{\ensuremath{N_{#1}}\xspace} \newcommand{\Mui}{\ensuremath{\mu_{\iter}}\xspace} \newcommand{\Nmin}{\ensuremath{N^\text{min}}\xspace} \newcommand{\Nsurv}{\ensuremath{N^\text{surv}}\xspace} \newcommand{\Nelite}{\ensuremath{N^\text{elite}}\xspace} \newcommand{\Nnew}{\ensuremath{N^\text{new}}\xspace} \newcommand{\bmax}{\ensuremath{b^\text{max}}\xspace} \newcommand{\bmin}{\ensuremath{b^\text{min}}\xspace} \newcommand{\Celite}{\ensuremath{\Theta^\text{elite}}\xspace} \ifthenelse {\boolean{Release}}{% \newcommand{\MANUEL}[1]{} \newcommand{\LESLIE}[1]{} \newcommand{\THOMAS}[1]{} }{% \newcommand{\MANUEL}[1]{{\footnotesize\noindent\textbf{\color{red}[~MANUEL: #1~]}}} \newcommand{\LESLIE}[1]{\footnote{\noindent\textbf{[ LESLIE: #1 ]}}} \newcommand{\THOMAS}[1]{\footnote{\noindent\textbf{[ THOMAS: #1 ]}}} } \newcommand{\hide}[1]{} \usepackage{tcolorbox} \newcommand{\infoicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{light-bulb-icon}}% \hspace{1em}} \newcommand{\warningicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{Warning-icon}}% \hspace{1em}} \definecolor{LightGray}{RGB}{193,193,193} \definecolor{LightYellow}{RGB}{253,247,172} \newlength\macroiconwidth \newenvironment{xwarningbox}{% \setlength{\fboxrule}{3.0\fboxrule}% \setlength{\fboxsep}{0\fboxsep}% \begin{tcolorbox}[colback=LightYellow,colframe=LightGray,boxrule=\fboxrule,boxsep=\fboxsep]% \infoicon% \settowidth{\macroiconwidth}{\infoicon}% \begin{minipage}[c]{\columnwidth - \macroiconwidth - 2.0\fboxrule - 2.0\fboxsep} \raggedright\footnotesize % }{% \end{minipage} \end{tcolorbox} % } % Workaround for broken knitr: https://github.com/yihui/knitr-examples/blob/master/036-latex-if.tex \providecommand{\hldef}[1]{\textcolor[rgb]{0.345,0.345,0.345}{#1}}% \providecommand{\hlsng}[1]{\textcolor[rgb]{0.192,0.494,0.8}{#1}}% <>= library(knitr) knit_hooks$set(inline = function(x) { if (is.numeric(x)) return(knitr:::format_sci(x, 'latex')) highr::hi_latex(x) }) @ \begin{document} <>= library(knitr) @ \author{Manuel L\'opez-Ib\'a\~nez, Leslie P\'erez C\'aceres, J\'er\'emie Dubois-Lacoste,\\ Thomas St\"utzle and Mauro Birattari \\IRIDIA, CoDE, Universit\'e Libre de Bruxelles, Brussels, Belgium} \title{The \irace Package: User Guide} \date{Version \iraceversion, \today} %\keywords{automatic % algorithm configuration, racing, parameter tuning, \aR} \maketitle \tableofcontents %Load files needed for examples <>= library("irace") load("examples.Rdata") # loads "experiment" and "output" iraceResults <- irace::read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)) log_ablation_file <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork = TRUE) load(log_ablation_file) options(width = 70) @ \newpage %% %% %% %% General info %% %% %% \section{General information} \MANUEL{Some things could be taken from the intro of the irace paper and reformulated.} \MANUEL{It would be good to mention that not only opt algorithms can be configured with irace, we say this in the paper.} \subsection{Background} \MANUEL{I would add a paragraph defining what is irace (a bit longer than the abstract above) and references to the literature so people can find more info. The first reference should be the irace TR.} \LESLIE{Here i guess we should say why tune an algorithm is a good idea, and why using irace is a better one.} The \irace package implements an \emph{iterated racing} procedure, which is an extension of Iterated F-race (\IFRACE)~\cite{BirYuaBal2010:emaoa}. The main use of \irace is the automatic configuration of optimization and decision algorithms, that is, finding the most appropriate settings of an algorithm given a set of instances of a problem. However, it may also be useful for configuring other types of algorithms when performance depends on the used parameter settings. It builds upon the \pkg{race} package by Birattari and it is implemented in \aR. The \irace package is available from CRAN: % \begin{center} \url{https://cran.r-project.org/package=irace} \end{center} % More information about \irace is available at \url{https://mlopez-ibanez.github.io/irace}. \subsection{Version} The current version of the \irace package is \iraceversion. Previous versions of the package can also be found in the \href{https://cran.r-project.org/package=irace}{CRAN website}. The algorithm underlying the current version of \irace and its motivation are described by \citet{LopDubPerStuBir2016irace}. The \textbf{adaptive capping mechanism} available from version $3.0$ is described by \citet{PerLopHooStu2017:lion}. Details of the implementation before version 2.0 can be found in a previous technical report~\cite{LopDubStu2011irace}. % \begin{xwarningbox} Versions of \irace before 2.0 are not compatible with the file formats detailed in this document. \end{xwarningbox} \subsection{License} The \irace package is Copyright \copyright{} \the\year\ and distributed under the GNU General Public License version 3.0 (\url{http://www.gnu.org/licenses/gpl-3.0.en.html}). The \irace package is free software (software libre): You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The \irace package is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. Please be aware that the fact that this program is released as Free Software does not excuse you from scientific propriety, which obligates you to give appropriate credit! If you write a scientific paper describing research that made substantive use of this program, it is your obligation as a scientist to (a) mention the fashion in which this software was used in the Methods section; (b) mention the algorithm in the References section. The appropriate citation is: \begin{itemize}[leftmargin=3em] \item[] Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Leslie Pérez Cáceres, Thomas Stützle, and Mauro Birattari. The \irace package: Iterated Racing for Automatic Algorithm Configuration. \emph{Operations Research Perspectives}, 3:43--58, 2016. doi:~\href{http://dx.doi.org/10.1016/j.orp.2016.09.002}{10.1016/j.orp.2016.09.002} \end{itemize} \section{Before starting} \MANUEL{I think this could be a bit more detailed by defining what is a parameter, a configuration, an instance, etc. but ok for now.} The \irace package provides an automatic configuration tool for tuning optimization algorithms, that is, automatically finding good configurations for the parameters values of a (target) algorithm saving the effort that normally requires manual tuning. \begin{figure}[t] \centering \includegraphics[width=0.6\textwidth]{irace-scheme} \caption{Scheme of \irace flow of information.} \label{fig:irace-scheme} \end{figure} Figure~\ref{fig:irace-scheme} gives a general scheme of how \irace works. \Irace receives as input a \emph{parameter space definition} corresponding to the parameters of the target algorithm that will be tuned, a set of \emph{instances} for which the parameters must be tuned for and a set of options for \irace that define the \emph{configuration scenario}. Then, \irace searches in the parameter search space for good performing algorithm configurations by executing the target algorithm on different instances and with different parameter configurations. A \parameter{targetRunner} must be provided to execute the target algorithm with a specific parameter configuration ($\theta$) and instance ($i$). The \parameter{targetRunner} function (or program) acts as an interface between the execution of the target algorithm and \irace: It receives the instance and configuration as arguments and must return the evaluation of the execution of the target algorithm. The following user guide contains guidelines for installing \irace, defining configuration scenarios, and using \irace to automatically configure your algorithms. %% %% %% %% Installation %% %% %% \section{Installation} \subsection{System requirements} \begin{itemize} \item \aR ($\text{version} \geq 3.2.0$) is required for running irace, but you don't need to know the \aR language to use it. \aR is freely available and you can download it from the \aR project website (\url{https://www.r-project.org}). See \autoref{sec:installation} for a quick installation guide of \aR. \item For GNU/Linux and OS X, the command-line executable \code{parallel-irace} requires GNU Bash. Individual examples may require additional software. \end{itemize} \subsection{\irace installation} \label{sec:irace install} The \irace package can be installed automatically within \aR or by manual download and installation. We advise to use the automatic installation unless particular circumstances do not allow it. The instructions to install \irace with the two mentioned methods are the following: \subsubsection[Install automatically within R]{Install automatically within \aR{}} Execute the following line in the \aR console to install the package: <>= install.packages("irace") @ Select a mirror close to your location, and test the installation in the \aR console with: <>= library("irace") q() # To exit R @ Alternatively, within the \aR graphical interface, you may use the \code{Packages and data->Package installer} menu on OS X or the \code{Packages} menu on Windows. \subsubsection{Manual download and installation} From the \irace package CRAN website (\url{https://cran.r-project.org/package=irace}), download one of the three versions available depending on your operating system: \begin{itemize} \item \code{irace_\iraceversion.tar.gz} (Unix/BSD/GNU/Linux) \item \code{irace_\iraceversion.tgz} (OS X) \item \code{irace_\iraceversion.zip} (Windows) \end{itemize} To install the package on GNU/Linux and OS X, you must execute the following command at the shell (replace \code{} with the path to the downloaded file, either \code{irace_\iraceversion.tar.gz} or \code{irace_\iraceversion.zip}): % \begin{lstlisting}[style=BashInputStyle] R CMD INSTALL \end{lstlisting} To install the package on Windows, open \aR and execute the following line on the \aR console (replace \code{} with the path to the downloaded file \code{irace_\iraceversion.zip}): %\LESLIE{Check that this actually works on internet says that this: \code{Rscript -e "install.packages('foo.zip', repos = NULL)"} also works} % <>= install.packages("", repos = NULL) @ If the previous installation instructions fail because of insufficient permissions and you do not have sufficient admin rights to install \irace system-wide, then you need to force a local installation. \subsubsection{Local installation} Let's assume you wish to install \irace on a path denoted by \code{}, which is a filesystem path for which you have sufficient rights. This directory \textbf{must} exist before attempting the installation. Moreover, you must provide to \aR the path to this library when loading the package. However, the latter can be avoided by adding the path to the system variable \code{R_LIBS} or to the \aR internal variable \code{.libPaths}, as we will see below.\footnote{% On Windows, see also \url{https://cran.r-project.org/bin/windows/base/rw-FAQ.html\#I-don_0027t-have-permission-to-write-to-the-R_002d3_002e3_002e1_005clibrary-directory}.} On GNU/Linux or OS X, execute the following commands to install the package on a local directory: \begin{lstlisting}[style=BashInputStyle] export R_LIBS_USER="" # Create R_LIBS_USER if it doesn't exist mkdir $R_LIBS_USER # Replace with the path to the downloaded file. R CMD INSTALL --library=$R_LIBS_USER # Tell R where to find R_LIBS_USER export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} On Windows, you can install the package on a local directory by executing the following lines in the \aR console: <>= # Replace with the path to the downloaded file. # Replace with the path used for installation. install.packages("", repos = NULL, lib = "") # Tell R where to find R_LIBS_USER. # This must be executed for every new session. .libPaths(c("", .libPaths())) @ \subsubsection{Testing the installation and invoking irace} Once \irace has been installed, load the package and test that the installation was successful by opening an \aR console and executing: <>= # Load the package library("irace") # Obtain the installation path system.file(package = "irace") @ The last command must print out the filesystem path where \irace is installed. In the remainder of this guide, the variable \code{\$IRACE_HOME} is used to denote this path. When executing any provided command that includes the \code{\$IRACE_HOME} variable do not forget to replace this variable with the installation path of \irace. On GNU/Linux or OS X, you can let the operating system know where to find \irace by defining the \code{\$IRACE_HOME} variable and adding it to the system \code{PATH}. Append the following commands to \path{~/.bash_profile}, \path{~/.bashrc} or \path{~/.profile}: % %<>= \begin{lstlisting}[style=BashInputStyle] # Replace with the irace installation path export IRACE_HOME= export PATH=${IRACE_HOME}/bin/:$PATH # Tell R where to find R_LIBS_USER # Use the following line only if local installation was forced export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} %@ Then, open a new terminal and launch \irace as follows: %<>= \begin{lstlisting}[style=BashInputStyle] irace --help \end{lstlisting} %@ On Windows, you need to add both \aR and the installation path of \irace to the environment variable \code{PATH}. To edit the \code{PATH}, search for ``Environment variables'' in the control panel, edit \code{PATH} and add a string similar to \path{C:\R_PATH\bin;C:\IRACE_HOME\bin\x64\} where \code{R_PATH} is the installation path of \aR and \code{IRACE_HOME} is the installation path of \irace. If \irace was installed locally, you also need to edit the environment variable \code{R_LIBS} to add \code{R_LIBS_USER}. Then, open a new terminal (run program \code{cmd.exe}) and launch \irace as: % %<>= \begin{lstlisting}[style=BashInputStyle] irace.exe --help \end{lstlisting} %@ Alternatively, you may directly invoke \irace from within the \aR console by executing: <>= library("irace") irace_cmdline("--help") @ \section{Running irace}\label{sec:execution} Before performing the tuning of your algorithm, it is necessary to define a tuning scenario that will give \irace all the necessary information to optimize the parameters of the algorithm. The tuning scenario is composed of the following elements: \begin{enumerate} \item Target algorithm parameter description (see \autoref{sec:target parameters}). \item Target algorithm runner (see \autoref{sec:runner}). \item Training instances list (see \autoref{sec:training}) \item \irace options (see \autoref{sec:irace options}). \item \textit{Optional:} Initial configurations (see \autoref{sec:initial}). \item \textit{Optional:} Target algorithm evaluator (see \autoref{sec:evaluator}). \end{enumerate} These scenario elements can be provided as plain text files or as \aR objects. This user guide provides examples of both types, but we advise the use of plain text files, which we consider the simpler option. For a step-by-step guide to create the scenario elements for your target algorithm continue to \autoref{sec:step}. For an example execution of \irace using the \ACOTSP scenario go to \autoref{sec:example}. \subsection{Step-by-step setup guide}\label{sec:step} This section provides a guide to setup a basic execution of \irace. The template files provided in the package (\IRACEHOME{/templates}) will be used as basis for creating your new scenario. Please follow carefully the indications provided in each step and in the template files used; if you have doubts check the the sections that describe each option in detail. \begin{enumerate}[leftmargin=*] \item Create a directory (\eg~\path{./tuning/}) for the scenario setup. This directory will contain all the files that describe the scenario. On GNU/Linux or OS X, you can do this as follows: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning \end{lstlisting} %@ \item Initialize the tuning directory with template config files. On GNU/Linux or OS X, you can do this as follows: %<>= \begin{lstlisting}[style=BashInputStyle] irace --init \end{lstlisting} %@ \item Define the target algorithm parameters to be tuned by following the instructions in \code{parameters.txt}. Available parameter types and other guidelines can be found in \autoref{sec:target parameters}. \item \textit{Optional}: Define the initial parameter configuration(s) of your algorithm, which allows you to provide good starting configurations (if you know some) for the tuning. Follow the instructions in \code{configurations.txt} and set \parameter{configurationsFile}\code{="configurations.txt"} in \code{scenario.txt}. More information in \autoref{sec:initial}. If you do not need to define initial configurations remove this file from the directory. \item Place the instances you would like to use for the tuning of your algorithm in the folder \path{./tuning/Instances/}. In addition, you can create a file (\eg~\code{instances-list.txt}) that specifies which instances from that directory should be run and which instance-specific parameters to use. To use such an instance file, set the appropriate option in \code{scenario.txt}, e.g., \parameter{trainInstancesFile} \code{= "instances-list.txt"}. See \autoref{sec:training} for guidelines. \item Uncomment and assign in \code{scenario.txt} only the options for which you need a value different from the default. Some common options that you might want to adjust are: \begin{description} \item[\parameter{execDir}] (\code{--exec-dir}): the directory in which \irace will execute the target algorithm; the default value is the current directory. %\item[\parameter{logFile}] (\code{--log-file}): a file the name of the results \aR data file that produces \irace. \item[\parameter{maxExperiments}] (\code{--max-experiments}): the maximum number of executions of the target algorithm that \irace will perform. \item[\parameter{maxTime}] (\code{--max-time}): maximum total execution time in seconds for the executions of \code{targetRunner}. In this case, \code{targetRunner} must return two values: cost and time. Note that you must provide either \parameter{maxTime} or \parameter{maxExperiments}. \item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set to \path{./Instances} if you put the training instances in that folder as instructed above. \end{description} For setting the tuning budget, see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}. \item Modify the \code{target-runner} script to run your algorithm. This script must execute your algorithm with the parameters and instance specified by \irace and return the evaluation of the execution and \textit{optionally} the execution time (\code{cost [time]}). When the \parameter{maxTime} option is used, returning \code{time} is mandatory. The \code{target-runner} template is written in \proglang{GNU Bash} scripting language, which can be executed easily in GNU/Linux and OS X systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. An example using Julia is available at \url{https://github.com/sbomsdorf/An-example-of-irace-using-Julia-code}. % Follow these instructions to adjust the given \code{target-runner} template to your algorithm: \begin{enumerate} \item Set the \code{EXE} variable with the path to the executable of the target algorithm. \item Set the \code{FIXED_PARAMS} if you need extra arguments in the execution line of your algorithm. An example could be the time that your algorithm is required to run (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--time 60"}) or the number of evaluations required (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--evaluations 10000"}). \item The line provided in the template executes the executable described in the \code{EXE} variable. % \begin{center} \code{\$EXE \$\{FIXED_PARAMS\} -i \$\{INSTANCE\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} % You must change this line according to the way your algorithm is executed. In this example, the algorithm receives the instance to solve with the flag \code{-i} and the seed of the random number generator with the flag \code{--seed}. The variable \code{CONFIG_PARAMS} adds to the command line the parameters that \irace has given for the execution. You must set the command line execution as needed. For example, the instance might not need a flag and might need to be the first argument: \begin{center} \code{\$EXE \$\{INSTANCE\} \$\{FIXED_PARAMS\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} The output of your algorithm is saved to the file defined in the \code{\$STDOUT} variable, and error output is saved in the file given by \code{\$STDERR}. The line: \begin{center} \code{if [ -s "\${STDOUT}" ]; then} \end{center} checks if the file containing the output of your algorithm is not empty. The example provided in the template assumes that your algorithm prints in the last output line the best result found (only a number). The line: \begin{center} \code{COST=\$(cat \$\{STDOUT\} | grep -e '\^{}[[:space:]]*[+-]\textbackslash{}?[0-9]' | cut -f1)} \end{center} parses the output of your algorithm to obtain the result from the last line. The \code{target-runner} script must print \textbf{only} one number. In the template example, the result is printed with \code{echo "\$COST"} (assuming \parameter{maxExperiments} is used) and the generated files are deleted (you may remove that line if you wish to keep them). \begin{xwarningbox} The \code{target-runner} script must be an executable file, unless you specify \parameter{targetRunnerLauncher}. \end{xwarningbox} You can test the target runner from the \aR console by checking the scenario as explained earlier in \autoref{sec:execution}. If you have problems related to the \code{target-runner} script when executing \irace, see \autoref{sec:check list} for a check list to help diagnose common problems. For more information about the \parameter{targetRunner}, please see \autoref{sec:runner}, \end{enumerate} \item \textit{Optional}: Modify the \code{target-evaluator} file. This is rarely needed and the \code{target-runner} template does not use it. \autoref{sec:evaluator} explains when a \parameter{targetEvaluator} is needed and how to define it. \item The \irace executable provides an option (\parameter{--check}) to check that the scenario is correctly defined. We recommend to perform a check every time you create a new scenario. When performing the check, \irace will verify that the scenario and parameter definitions are correct and will test the execution of the target algorithm. To check your scenario execute the following commands: \begin{itemize} \item From the command-line (on Windows, execute \code{irace.bat}): %<>= \begin{lstlisting}[style=BashInputStyle] # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace --scenario scenario.txt --check \end{lstlisting} %@ \item Or from the \aR console: <>= library("irace") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) checkIraceScenario(scenario = scenario) @ \end{itemize} \item Once all the scenario elements are prepared you can execute \irace, either using the command-line wrappers provided by the package or directly from the \aR console: \begin{itemize} \item \textbf{From the command-line console}, call the command (on Windows, you should execute \code{irace.exe}): \begin{lstlisting}[style=BashInputStyle] cd ./tuning/ # $IRACE_HOME is the installation directory of irace # By default, irace reads scenario.txt, you can specify a different file # with --scenario. $IRACE_HOME/bin/irace \end{lstlisting} For this example, we assume that the needed scenario files have been set properly in the \code{scenario.txt} file using the options described in \autoref{sec:irace options}. Most \irace options can be specified in the command line or directly in the \code{scenario.txt} file. \item \textbf{From the \aR console}, evaluate: <>= library("irace") # Go to the directory containing the scenario files setwd("./tuning") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) irace_main(scenario = scenario) @ \end{itemize} This will perform one run of \irace. See the output of \code{irace --help} in the command-line or \code{irace_cmdline("--help")} in \aR for quick information on additional \irace options. For more information about \irace options, see \autoref{sec:irace options}. \end{enumerate} \begin{xwarningbox} Command-line options override the same options specified in the \code{scenario.txt} file. \end{xwarningbox} \subsection{Setup example for ACOTSP}\label{sec:example} The \ACOTSP tuning example can be found in the package installation in the folder \IRACEHOME{/examples/acotsp}. % Other example scenarios can be found in the same folder. More examples of tuning scenarios can be found in the Algorithm Configuration Library (AClib, \url{http://www.aclib.net/}). In this section, we describe how to execute the \ACOTSP scenario. If you wish to start setting up your own scenario, continue to the next section. For this example, we assume a GNU/Linux system such as Ubuntu with a working \proglang{C} compiler such as \code{gcc}. To execute this scenario follow these steps: \begin{enumerate} \item Create a directory for the tuning (\eg~\path{./tuning/}) and copy the example scenario files located in the \code{examples} folder to the created directory: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning # $IRACE_HOME is the installation directory of irace. cp $IRACE_HOME/examples/acotsp/* ./ ls ./ # Make sure that target-runner is executable chmod u+x target-runner \end{lstlisting} %@ \item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz} to the \path{./tuning/} directory and decompress it, which creates create a folder \path{instances}: %<>= \begin{lstlisting}[style=BashInputStyle] tar -xvf instances.tar.gz ls instances/ \end{lstlisting} %@ If the above gives an error or does not show any files, then the files were not extract correctly. Maybe the \path{instances.tar.gz} file did not download correctly or maybe it is not in the correct place. It should be within the folder \path{tuning}.correctly. \item Download the \ACOTSP software from \url{https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip} to the \path{./tuning/} directory and compile the \path{acotsp} executable using \code{make}. %<>= \begin{lstlisting}[style=BashInputStyle] unzip master.zip make -C ACOTSPQAP-master acotsp ./ACOTSPQAP-master/acotsp --version \end{lstlisting} %@ If the above gives an error, then the \path{acotsp} executable failed to compile for some reason. Maybe you are missing the C compiler or some files did not extract correctly. \item Create a directory for executing the experiments and execute \irace: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./acotsp-arena/ # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace \end{lstlisting} %@ Or you can also execute \irace from the \aR console using: <>= library("irace") setwd("./tuning/") irace_cmdline() @ \end{enumerate} The most usual sources of error when running the above commands are: \begin{itemize} \item The \irace package is not correctly installed. Please make sure that installing \irace did not give any errors. \item The location of the files is not correct. Please make sure that you have: \begin{itemize} \item The folder \path{tuning} and that it contains the files \path{scenario.txt}, \path{parameters-acotsp.txt}, \path{target-runner}, and the folders \path{instances}, \path{ACOTSPQAP-master} and \path{acotsp-arena}. \item The folder \path{instances} should contain the TSP instance files (\path{*.tsp}). \item The folder \path{ACOTSPQAP-master} should contain the executable \path{acotsp}. You should be able to invoke \code{./ACOTSPQAP-master/acotsp --version} without an error. \item The folder \path{acotsp-arena} should be empty. \end{itemize} \end{itemize} %% %% %% %% Scenario settings %% %% %% \section{Defining a configuration scenario}\label{sec:scenario} \subsection{Target algorithm parameters} \label{sec:target parameters} The parameters of the target algorithm are defined by a parameter file as described in \autoref{sec:parameters file}. Optionally, when executing \irace from the \aR console, the parameters can be specified directly as an \aR object (see \autoref{sec:parameters object}). For defining your parameters follow the guidelines provided in the following sections. \subsubsection{Parameter types} Each target parameter has an associated type that defines its domain and the way \irace handles them internally. Understanding the nature of the domains of the target parameters is important to select appropriate types. The four basic types supported by \irace are the following: \begin{itemize} \item \textit{Real} parameters are numerical parameters that can take floating-point values within a given range. The range is specified as an interval `\code{(,)}'. This interval is closed, that is, the parameter value may eventually be one of the bounds. The possible values are rounded to a number of \emph{decimal places} specified by the global option \code{digits} (Section~\ref{sec:globaloptions}). For example, given the default number of digits of $4$, the values $0.12345$ and $0.12341$ are both rounded to $0.1234$. Selected real-valued parameters can be optionally sampled on a logarithmic scale (base $e$). \item \textit{Integer} parameters are numerical parameters that can take only integer values within the given range. Their range is specified as the range of real parameters and they can also be optionally sampled on a logarithmic scale (base $e$). \item \textit{Categorical} parameters are defined by a set of possible values specified as `\code{(,} \code{...,} \code{)}'. The values are quoted or unquoted character strings. Empty strings and strings containing commas or spaces must be quoted. \item \emph{Ordinal} parameters are defined by an \emph{ordered} set of possible values in the same format as for categorical parameters. They are handled internally as integer parameters, where the integers correspond to the indexes of the values. \end{itemize} \begin{xwarningbox} Boolean (or logical) parameters are best encoded as categorical ones with just two values rather than integer ones with domain $(0,1)$. Some boolean parameters take an explicit value (0/1 or true/false) such as: \begin{CodeInput} dlb "--dlb " c (0, 1) \end{CodeInput} Others are switches whose presence activates the parameter: \begin{CodeInput} dlb "" c ("", "--dlb") \end{CodeInput} \end{xwarningbox} \subsubsection{Parameter domains} For each target parameter, an interval or a set of values must be defined according to its type, as described above. There is no limit for the size of the set or the length of the interval, but keep in mind that larger domains could increase the difficulty of the tuning task. Choose always values that you consider relevant for the tuning. In case of doubt, we recommend to choose larger intervals, as occasionally best parameter settings may be not intuitive a priori. All intervals are considered as closed intervals. It is possible to define parameters that will have always the same value. Such ``\emph{fixed}'' parameters will not be tuned but their values are used when executing the target algorithm and they are affected by constraints defined on them. All fixed parameters must be defined as categorical parameters and have a domain of one element. \subsubsection{Parameter dependent domains}\label{sec:paramdependant} Domains that are dependent on the values of other parameters can be specified only for numerical parameters (both integer and real). To do so, the dependent domain must be expressed in function of another parameter, which must be a numerical parameter. The expression that defines a dependency must be written between quotes: \code{(value, "expression")} or \code{("expression", value)} or \code{("expression", "expression")}. The expressions can only use the following operators and \aR functions: \code{+}, \code{-}, \code{*}, \code{/}, \code{\%\%}, \code{min}, \code{max}, \code{round}, \code{floor}, \code{ceiling}, \code{trunc}. If you need to use an operator or function not listed here, please contact us. \begin{xwarningbox} The user must ensure that the defined domain is valid at all times since \irace currently is not able to detect possible invalid domains based on the expressions provided. \end{xwarningbox} If you have a parameter \code{p2} that is just a transformation of another \code{p1}, then instead of using a dependent domain (left-hand side of the following example), it will be better to create a dummy parameter that controls the transformation (right-hand side) and do the transformation within \code{target-runner}. For example: % \begin{center} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With dependent domains p1 "" r (0, 100) p2 "" r ("p1", "p1 + 10") \end{CodeInput} \end{minipage} \hspace{\stretch{1}} should be \hspace{\stretch{1}} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With a dummy parameter p1 "" r (0, 100) p2dum "" r (0, 10) \end{CodeInput} \end{minipage} \end{center} % and \code{target-runner} will compute $\code{p2} = \code{p2dum} \cdot \code{p1}$. \subsubsection{Conditional parameters}\label{sec:conditional} Conditional parameters are active only when others have certain values. These dependencies define a hierarchical relation between parameters. For example, the target algorithm may have a parameter \code{localsearch} that takes values \code{(sa, ts)} and another parameter \code{ts-length} that only needs to be set if the first parameter takes precisely the value \code{ts}. Thus, parameter \code{ts-length} is conditional on \code{localsearch == "ts"}. \subsubsection{Forbidden parameter configurations}\label{sec:forbidden} A line containing just \texttt{[forbidden]} ends the list of parameters and starts the list of forbidden expressions. Each line is a logical expression (in \aR syntax) containing parameter names as defined by the \parameter{parameterFile} (\autoref{sec:target parameters}), values and logical operators. For a list of \aR logical operators see: \begin{center} \url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html} \end{center} If \irace generates a parameter configuration that makes any of the logical expressions evaluate to \code{TRUE}, then the configuration is considered forbidden and it is never evaluated. This is useful when some combination of parameter values could cause the target algorithm to crash, consume excessive CPU time or memory, or when it is known that they do no produce satisfactory results. \begin{xwarningbox} Initial configuration (\autoref{sec:initial}) that are forbidden will be discarded with a warning. \end{xwarningbox} If the forbidden constraints provided are too strict, \irace may produce the following error: \begin{CodeInput} irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict? \end{CodeInput} % In that case, it may be a good idea to reformulate the forbidden constraints as conditional parameters (\autoref{sec:conditional}), parameter-dependent domains (\autoref{sec:paramdependant}), repairing the configurations (\autoref{sec:repairconf}) or post-processing within the target-algorithm (\autoref{sec:complex_domains}). \subsubsection{Global options}\label{sec:globaloptions} A line containing just \texttt{[global]} starts the definition of global options. The only global option currently implemented is \code{digits}, which controls the number of decimal digits for real valued parameters. Its default value is 4. \subsubsection{Parameter file format}\label{sec:parameters file} For simplicity, the description of the parameters space is given as a table. Each line of the table defines a configurable parameter % \begin{center} \code{\