Unable to access config file -- Unknown host: java.net.UnknownHostException: api.tower.nf
NickSwainston opened this issue · 2 comments
NickSwainston commented
I'm trying to run nf-tower using a SLURM platform. It fails as it can not download an SCM config file even though I haven't set any secrets. I have checked that I can access api.tower.nf
with the curl
command and I can run nextflow -with-tower
commands on the nodes I am trying to use as the head node. Here is an example of the launch.sh:
#!/bin/bash
#SBATCH -D /fred/oz005/users/nswainst/tower
#SBATCH -J nf-workflow-1k0K4WlGep8ZTM
#SBATCH -o /fred/oz005/users/nswainst/tower/nf-1k0K4WlGep8ZTM.log
#SBATCH --no-requeue
#SBATCH -p trevor
#SBATCH --mem=8GB
set -e
set -o pipefail
# Input variables:
#
# - NXF_UUID: nextflow session id generated by tower
# - NXF_LOG_FILE: nextflow log file name
# - NXF_OUT_FILE: nextflow output file name
# - NXF_IGNORE_RESUME_HISTORY: do not stop for missing nextflow history file
# - NXF_CONFIG_BASE64: nextflow config file encoded as base64 string
# - NXF_SCM_BASE64: nextflow scm file encoded as base64 string
# - NXF_DEBUG: enable debugging mode
# - TOWER_ACCESS_TOKEN: Tower access token
# - TOWER_WORKFLOW_ID: Workflow ID generated by Tower
# - TOWER_CONFIG_BASE64: tower config file encoded as base64 string
# - TOWER_CONFIG_FILE: tower config file name
export NXF_IGNORE_RESUME_HISTORY=true
export NXF_WORK=/fred/oz005/users/nswainst/work
export TOWER_REPORTS_FILE=nf-1k0K4WlGep8ZTM-reports.tsv
export NXF_EXIT_FILE=nf-1k0K4WlGep8ZTM.exit
export NXF_CONFIG_FILE=nf-1k0K4WlGep8ZTM.config
export NXF_OUT_FILE=nf-1k0K4WlGep8ZTM.txt
export NXF_ASSETS=/fred/oz005/users/nswainst/work/.nextflow/pipelines/56a2786f
export NXF_UUID=40f3f3fb-db1e-4736-a606-8beb680628b1
export NXF_TML_FILE=timeline-1k0K4WlGep8ZTM.html
export TOWER_WORKFLOW_ID=1k0K4WlGep8ZTM
export NXF_ANSI_LOG=false
export NXF_PLUGINS_DEFAULT=nf-tower
export NXF_PRERUN_BASE64=bW9kdWxlIHVzZSAvYXBwcy91c2Vycy9wdWxzYXIvb3BlbnN0YWNrL2djYy0xMS4zLjAvbW9kdWxlZmlsZXMKbW9kdWxlIGxvYWQgcHNyaG9tZS9sYXRlc3QKZXhwb3J0IFRPV0VSX0FDQ0VTU19UT0tFTj1leUpoYkdjaU9pSklVekkxTmlKOS5leUp6ZFdJaU9pSTRNak0wSWl3aWJtSm1Jam94Tmprd09UWTFOelEyTENKeWIyeGxjeUk2V3lKMWMyVnlJbDBzSW1semN5STZJblJ2ZDJWeUxXRndjQ0lzSW1WNGNDSTZNVFk1TURrMk9UTTBOaXdpYVdGMElqb3hOamt3T1RZMU56UTJmUS4xdnBBaDdpUU1rT3pjU1p1ZnF1RmphWWlIbDVNNnhRNE1Ba2RTeVhMMkNzCmV4cG9ydCBUT1dFUl9SRUZSRVNIX1RPS0VOPWV5SmhiR2NpT2lKSVV6STFOaUo5Lk56RmtZVFU0TXpRdE5XRTJNQzAwT0dRNExUZ3daR0l0WXpka1pqVXlZakF3WXpGaS40STl1a3FfWWU5enNPZWoxTFczVEpNb0Q4VTZWaGZheGR2UUdpZWVOcjRNCmV4cG9ydCBOWEZfU0NNX0ZJTEU9aHR0cHM6Ly9hcGkudG93ZXIubmYvZXBoZW1lcmFsL3hERndOSkpDWWhoNUpmSERMakp3VGcK
export NXF_LOG_FILE=nf-1k0K4WlGep8ZTM.log
export NXF_CONFIG_BASE64=dGltZWxpbmUuZW5hYmxlZCA9IHRydWUKdGltZWxpbmUuZmlsZSA9ICIkTlhGX1RNTF9GSUxFIgpwcm9jZXNzLmV4ZWN1dG9yID0gJ3NsdXJtJwpwcm9jZXNzLnF1ZXVlID0gJ21pbGFuJwp3b3JrRGlyID0gJy9mcmVkL296MDA1L3VzZXJzL25zd2FpbnN0L3dvcmsnCi8vLS0tIHVzZXIgY3VzdG9tIGNvbmZpZwppbmNsdWRlQ29uZmlnICdodHRwczovL2FwaS50b3dlci5uZi9lcGhlbWVyYWwvc0h5MV9LY3pDaGxMLThNVmVWMkpOQSc=
[[ $NXF_DEBUG ]] && (env | sort) && set -x
cache_path=".nextflow/cache/$NXF_UUID"
function save_exit() {
# Save exit code to file: note NXF_EXIT_FILE is expected to always be set; otherwise, the script will fail (return a non-zero exit code) at this point.
[[ $NXF_EXIT_FILE ]] && printf $1 > $NXF_EXIT_FILE
}
function pre_run() {
if [[ $NXF_PRERUN_BASE64 ]]; then
source /dev/stdin <<<"$(cat <(echo $NXF_PRERUN_BASE64 | base64 -d))" > >(tee -a $NXF_OUT_FILE) 2>&1
fi
}
function post_run() {
if [[ $NXF_POSTRUN_BASE64 ]]; then
bash <(echo $NXF_POSTRUN_BASE64 | base64 -d) > >(tee -a $NXF_OUT_FILE) 2>&1 || true
fi
}
function on_exit() {
NXF_EXIT_STATUS=$?
save_exit $NXF_EXIT_STATUS
rm -rf $NXF_SECRETS_FILE
export NXF_EXIT_STATUS
post_run
exit $NXF_EXIT_STATUS
}
function load_cache() {
if [[ $TOWER_RESUME_DIR ]]; then
mkdir -p "$cache_path"
[ -e "$TOWER_RESUME_DIR/$cache_path" ] && rsync -r "$TOWER_RESUME_DIR/$cache_path"/ "$cache_path" || true
fi
}
function term_run() {
kill -TERM $nf_pid
wait $nf_pid
}
trap 'save_exit $?' EXIT
pre_run
load_cache
if [[ $NXF_CONFIG_BASE64 ]]; then
echo $NXF_CONFIG_BASE64 | base64 -d > ${NXF_CONFIG_FILE:-nextflow.config}
unset NXF_CONFIG_BASE64
fi
# save tower config file
if [[ $TOWER_CONFIG_BASE64 ]]; then
echo $TOWER_CONFIG_BASE64 | base64 -d > $TOWER_CONFIG_FILE
fi
# save secrets
if [[ $NXF_SECRETS_BASE64 ]]; then
export NXF_SECRETS_FILE=$PWD/nf-${TOWER_WORKFLOW_ID}.secrets.json
echo $NXF_SECRETS_BASE64 | base64 -d > $NXF_SECRETS_FILE
chmod 600 $NXF_SECRETS_FILE
fi
[[ $NXF_DEBUG ]] && nextflow -Dcapsule.log=verbose info -dd
trap term_run TERM INT USR2
trap on_exit EXIT
trap '' USR1
nextflow run https\://github.com/OZGrav/nf-core-meerpipe -name cranky_woese -params-file https\://api.tower.nf/ephemeral/H9W5NkAJHCIi-VgCVOO7LA.json -with-tower -r dev -latest > >(tee -a $NXF_OUT_FILE) 2>&1 &
nf_pid=$!
wait $nf_pid
and here is the log:
Aug-02 18:42:46.225 [main] DEBUG nextflow.cli.Launcher - $> nextflow run 'https://github.com/OZGrav/nf-core-meerpipe' -name cranky_woese -params-file 'https://api.tower.nf/ephemeral/H9W5NkAJHCIi-VgCVOO7LA.json' -with-tower -r dev -latest
Aug-02 18:42:46.261 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 22.10.6
N E X T F L O W ~ version 22.10.6
Aug-02 18:42:46.309 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; embedded=false; plugins-dir=/home/nswainst/.nextflow/plugins; core-plugins: nf-amazon@1.11.3,nf-azure@0.14.2,nf-codecommit@0.1.2,nf-console@1.0.4,nf-ga4gh@1.0.4,nf-google@1.4.5,nf-tower@1.5.6,nf-wave@0.5.3
Aug-02 18:42:46.337 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: []
Aug-02 18:42:46.338 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: []
Aug-02 18:42:46.341 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode
Aug-02 18:42:46.367 [main] INFO org.pf4j.AbstractPluginManager - No plugins
Aug-02 18:42:46.380 [main] DEBUG nextflow.scm.ProviderConfig - Detected SCM custom path: https://api.tower.nf/ephemeral/xDFwNJJCYhh5JfHDLjJwTg
Aug-02 18:42:46.641 [main] ERROR nextflow.cli.Launcher - Unable to access config file 'https://api.tower.nf/ephemeral/xDFwNJJCYhh5JfHDLjJwTg' -- Unknown host: java.net.UnknownHostException: api.tower.nf
api.tower.nf
java.net.UnknownHostException: api.tower.nf
at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:229)
at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.base/java.net.Socket.connect(Socket.java:609)
at java.base/sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:305)
at java.base/sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
at java.base/sun.net.NetworkClient.doConnect(NetworkClient.java:182)
at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:508)
at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:603)
at java.base/sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:266)
at java.base/sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:373)
at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:207)
at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1187)
at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1081)
at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:193)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1592)
at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1520)
at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:527)
at java.base/sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:334)
at nextflow.file.http.XFileSystemProvider.toConnection0(XFileSystemProvider.groovy:222)
at nextflow.file.http.XFileSystemProvider.toConnection(XFileSystemProvider.groovy:210)
at nextflow.file.http.XFileSystemProvider.newInputStream(XFileSystemProvider.groovy:359)
at java.base/java.nio.file.Files.newInputStream(Files.java:156)
at java.base/java.nio.file.Files.newBufferedReader(Files.java:2839)
at org.apache.groovy.nio.extensions.NioExtensions.newReader(NioExtensions.java:1404)
at org.apache.groovy.nio.extensions.NioExtensions.getText(NioExtensions.java:397)
at nextflow.scm.ProviderConfig.getFromFile(ProviderConfig.groovy:286)
at nextflow.scm.ProviderConfig.getDefault(ProviderConfig.groovy:332)
at nextflow.scm.AssetManager.<init>(AssetManager.groovy:103)
at nextflow.cli.CmdRun.getScriptFile0(CmdRun.groovy:503)
at nextflow.cli.CmdRun.getScriptFile(CmdRun.groovy:444)
at nextflow.cli.CmdRun.run(CmdRun.groovy:300)
at nextflow.cli.Launcher.run(Launcher.groovy:487)
at nextflow.cli.Launcher.main(Launcher.groovy:646)
Unknown network host: api.tower.nf
Any help you can provide helping me fix this bug would be greatly appreciated.
pditommaso commented
It looks like an network problem on your side, that's a valid host name
curl https://api.tower.nf/service-info
NickSwainston commented
Turns out the head Nextflow command job was being sent to the wrong partition (which doesn't have internet access), which is a separate issue.